uoa-repository-manager-service/src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java

1038 lines
45 KiB
Java

package eu.dnetlib.repo.manager.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.domain.data.Repository;
import eu.dnetlib.domain.data.RepositoryInterface;
import eu.dnetlib.domain.enabling.Vocabulary;
import eu.dnetlib.domain.functionality.validator.JobForValidation;
import eu.dnetlib.repo.manager.domain.*;
import eu.dnetlib.repo.manager.domain.dto.Role;
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService;
import eu.dnetlib.repo.manager.service.security.AuthoritiesUpdater;
import eu.dnetlib.repo.manager.service.security.AuthorizationService;
import eu.dnetlib.repo.manager.service.security.RoleMappingService;
import eu.dnetlib.repo.manager.utils.Converter;
import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.http.entity.ContentType;
import org.apache.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mitre.openid.connect.model.OIDCAuthenticationToken;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.util.MimeType;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponents;
import org.springframework.web.util.UriComponentsBuilder;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@Service("repositoryService")
public class RepositoryServiceImpl implements RepositoryService {
private static final Logger LOGGER = Logger.getLogger(RepositoryServiceImpl.class);
private final AuthorizationService authorizationService;
private final RoleMappingService roleMappingService;
private final AaiRegistryService registryCalls;
private final AuthoritiesUpdater authoritiesUpdater;
private final RestTemplate restTemplate;
private final VocabularyLoader vocabularyLoader;
private final PiWikService piWikService;
private final EmailUtils emailUtils;
private final ValidatorService validatorService;
@Value("${api.baseAddress}")
private String baseAddress;
@Value("${services.repo-manager.adminEmail}")
private String adminEmail;
@Value("${services.repomanager.usageStatisticsDiagramsBaseURL}")
private String usageStatisticsDiagramsBaseURL;
@Value("${services.repomanager.usageStatisticsNumbersBaseURL}")
private String usageStatisticsNumbersBaseURL;
private static final Map<String, List<String>> dataSourceClass = new HashMap<>();
private static final Map<String, String> invertedDataSourceClass = new HashMap<>();
private final String[] vocabularyNames = {"dnet:countries", "dnet:datasource_typologies", "dnet:compatibilityLevel"};
private final Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<>();
private final Map<String, String> countriesMap = new HashMap<>();
private final Map<String, String> inverseCountriesMap = new HashMap<>();
private HttpHeaders httpHeaders;
@Autowired
public RepositoryServiceImpl(AuthorizationService authorizationService,
RoleMappingService roleMappingService,
AaiRegistryService registryCalls,
AuthoritiesUpdater authoritiesUpdater,
VocabularyLoader vocabularyLoader,
RestTemplate restTemplate,
@Lazy EmailUtils emailUtils,
@Lazy ValidatorService validatorService,
@Lazy PiWikService piWikService) {
this.authorizationService = authorizationService;
this.roleMappingService = roleMappingService;
this.registryCalls = registryCalls;
this.authoritiesUpdater = authoritiesUpdater;
this.vocabularyLoader = vocabularyLoader;
this.piWikService = piWikService;
this.emailUtils = emailUtils;
this.validatorService = validatorService;
this.restTemplate = restTemplate;
}
private String getAuthenticatedUserEmail() {
OIDCAuthenticationToken authenticationToken = (OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
return authenticationToken.getUserInfo().getEmail();
}
@PostConstruct
private void init() {
LOGGER.debug("Initialization method of repository api!");
LOGGER.debug("Updated version!");
dataSourceClass.put("opendoar", Arrays.asList("pubsrepository::institutional", "pubsrepository::thematic", "pubsrepository::unknown", "pubsrepository::mock"));
dataSourceClass.put("re3data", Collections.singletonList("datarepository::unknown"));
dataSourceClass.put("journal", Collections.singletonList("pubsrepository::journal"));
dataSourceClass.put("aggregator", Arrays.asList("aggregator::pubsrepository::institutional", "aggregator::pubsrepository::journals", "aggregator::datarepository", "aggregator::pubsrepository::unknown"));
invertedDataSourceClass.put("pubsrepository::institutional", "opendoar");
invertedDataSourceClass.put("pubsrepository::thematic", "opendoar");
invertedDataSourceClass.put("pubsrepository::unknown", "opendoar");
invertedDataSourceClass.put("pubsrepository::mock", "opendoar");
invertedDataSourceClass.put("datarepository::unknown", "re3data");
invertedDataSourceClass.put("pubsrepository::journal", "journal");
invertedDataSourceClass.put("aggregator::pubsrepository::institutional", "aggregator");
invertedDataSourceClass.put("aggregator::pubsrepository::journals", "aggregator");
invertedDataSourceClass.put("aggregator::datarepository", "aggregator");
invertedDataSourceClass.put("aggregator::pubsrepository::unknown", "aggregator");
httpHeaders = new HttpHeaders();
httpHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8);
for (String vocName : vocabularyNames) {
vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
}
Country[] countries = getCountries();
for (Country c : countries) {
countriesMap.put(c.getName(), c.getCode());
inverseCountriesMap.put(c.getCode(), c.getName());
}
}
@Override
public Country[] getCountries() {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/countries")
.build().encode();
return restTemplate.getForObject(uriComponents.toUri(), Country[].class);
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<Repository> getRepositories(List<String> ids) throws JSONException {
return getRepositories(ids, 0, 10);
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<Repository> getRepositories(List<String> ids, int page, int size) throws JSONException {
List<Repository> repos = new ArrayList<>();
LOGGER.debug("Retreiving repositories with ids : " + String.join(", ", ids));
UriComponents uriComponents = searchDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
RequestFilter requestFilter = new RequestFilter();
for (String repoId : ids) {
requestFilter.setId(repoId);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
repos.addAll(Converter.jsonToRepositoryList(new JSONObject(rs)));
}
for (Repository r : repos)
r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
return repos;
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids) throws Exception {
return getRepositoriesSnippets(ids, 0, 10);
}
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
// another way for paging must be implemented.
@Override
public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids, int page, int size) throws Exception {
List<RepositorySnippet> resultSet = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
// here page should be 0
UriComponents uriComponents = searchSnipperDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
RequestFilter requestFilter = new RequestFilter();
try {
for (String repoId : ids) {
requestFilter.setId(repoId);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),
mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
}
} catch (Exception e) {
LOGGER.debug("Exception on getRepositoriesSnippetOfUser", e);
throw e;
}
LOGGER.debug("resultSet:" + resultSet);
resultSet.parallelStream().forEach(repositorySnippet -> {
repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId()));
});
return resultSet;
}
@Override
public List<RepositorySnippet> getRepositoriesByCountry(String country,
String mode,
Boolean managed) throws JSONException, IOException {
LOGGER.debug("Getting repositories by country!");
int page = 0;
int size = 100;
List<RepositorySnippet> resultSet = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
String filterKey = "UNKNOWN";
if (mode.equalsIgnoreCase("opendoar"))
filterKey = "openaire____::opendoar";
else if (mode.equalsIgnoreCase("re3data"))
filterKey = "openaire____::re3data";
LOGGER.debug("Country code equals : " + country);
LOGGER.debug("Filter mode equals : " + filterKey);
UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));
RequestFilter requestFilter = new RequestFilter();
requestFilter.setCountry(country);
requestFilter.setCollectedfrom(filterKey);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
while (jsonArray.length() > 0) {
LOGGER.debug("json: " + jsonArray);
resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),
mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
page += 1;
uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));
rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
}
return resultSet;
}
public List<RepositorySnippet> searchRegisteredRepositories(String country, String typology, String englishName,
String officialName, String requestSortBy, String order, int page, int pageSize) throws Exception {
LOGGER.debug("Searching registered repositories");
List<RepositorySnippet> resultSet = new ArrayList<>();
ObjectMapper mapper = new ObjectMapper();
UriComponents uriComponents = searchRegisteredDatasource(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize));
RequestFilter requestFilter = new RequestFilter();
requestFilter.setCountry(country);
requestFilter.setTypology(typology);
requestFilter.setOfficialname(officialName);
requestFilter.setEnglishname(englishName);
try {
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));
return resultSet;
} catch (Exception e) {
LOGGER.error("Error searching registered datasources", e);
throw e;
}
}
@Override
public int getTotalRegisteredRepositories() {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/countregistered")
.queryParam("fromDate", "1900-01-01")
.build().encode();
return restTemplate.getForObject(uriComponents.toUri(), Integer.class);
}
private Repository updateRepositoryInfo(Repository r) throws JSONException {
/*
* from datasource class
* we get the datasource type form the inverted map
* */
r.setDatasourceType(getRepositoryType(r.getDatasourceClass()));
r.setInterfaces(this.getRepositoryInterface(r.getId()));
r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
r.setCountryName(getCountryName(r.getCountryCode()));
return r;
}
private Collection<Repository> getRepositoriesByMode(String mode, List<Repository> rs) {
List<Repository> reps = new ArrayList<>();
for (Repository r : rs) {
if (r.getCollectedFrom() != null && r.getCollectedFrom().equals(mode))
reps.add(r);
}
return reps;
}
@Override
public List<Repository> getRepositoriesOfUser(String page, String size) throws JSONException {
String userEmail = ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail();
LOGGER.debug("Retreiving repositories of authenticated user : " + userEmail);
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());
return getRepositories(new ArrayList<>(repoIds));
}
@Override
public List<Repository> getRepositoriesOfUser(String userEmail, String page, String size) throws JSONException {
LOGGER.debug("Retreiving repositories of authenticated user : " + userEmail);
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));
return getRepositories(new ArrayList<>(repoIds));
}
@Override
public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String page, String size) throws Exception {
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());
return getRepositoriesSnippets(new ArrayList<>(repoIds));
}
@Override
public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String userEmail, String page, String size) throws Exception {
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));
return getRepositoriesSnippets(new ArrayList<>(repoIds));
}
@Override
public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException {
LOGGER.debug("Retreiving repositories with id : " + id);
RepositorySnippet repo = null;
UriComponents uriComponents = searchSnipperDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setId(id);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
if (jsonArray.length() == 0)
throw new ResourceNotFoundException();
repo = Converter.jsonToRepositorySnippetObject(jsonArray.getJSONObject(0));
return repo;
}
@Override
public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException {
LOGGER.debug("Retreiving repositories with id : " + id);
Repository repo = null;
UriComponents uriComponents = searchDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setId(id);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");
if (jsonArray.length() == 0)
throw new ResourceNotFoundException();
repo = Converter.jsonToRepositoryObject(jsonArray.getJSONObject(0));
return updateRepositoryInfo(repo);
}
@Override
public List<AggregationDetails> getRepositoryAggregations(String id, int from, int size) throws JSONException {
LOGGER.debug("Retreiving aggregations for repository with id : " + id);
UriComponents uriComponents = searchDatasource(from + "", size + "");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setId(id);
List<AggregationDetails> aggregationHistory = new ArrayList<>();
long start = System.currentTimeMillis();
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
long end = System.currentTimeMillis();
System.out.println("Aggregations request through rest template took " + (end - start) + "ms");
JSONObject repository = new JSONObject(rs);
if (repository.getJSONArray("datasourceInfo").length() == 0)
return aggregationHistory;
start = System.currentTimeMillis();
aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0)));
end = System.currentTimeMillis();
System.out.println("Getting aggregations history from json " + (end - start) + "ms");
return aggregationHistory.size() == 0 ? aggregationHistory : aggregationHistory.stream()
.sorted(Comparator.comparing(AggregationDetails::getDate).reversed())
.limit(size)
.collect(Collectors.toList());
}
@Override
public Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(String id) throws JSONException {
LOGGER.debug("Retreiving aggregations (by year) for repository with id : " + id);
UriComponents uriComponents = searchDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setId(id);
List<AggregationDetails> aggregationHistory = new ArrayList<>();
Map<String, List<AggregationDetails>> aggregationByYear = new HashMap<>();
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
JSONObject repository = new JSONObject(rs);
if (repository.getJSONArray("datasourceInfo").length() == 0)
return aggregationByYear;
aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0)));
return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory);
}
private Map<String, List<AggregationDetails>> createYearMap(List<AggregationDetails> aggregationHistory) {
Map<String, List<AggregationDetails>> aggregationByYear;
aggregationHistory = aggregationHistory.stream()
.sorted(Comparator.comparing(AggregationDetails::getDate).reversed())
.collect(Collectors.toList());
return aggregationHistory.stream()
.collect(Collectors.groupingBy(AggregationDetails::getYear));
}
@Override
public List<Repository> getRepositoriesByName(String name,
String page,
String size) throws JSONException {
LOGGER.debug("Retreiving repositories with official name : " + name);
UriComponents uriComponents = searchDatasource("0", "100");
RequestFilter requestFilter = new RequestFilter();
requestFilter.setOfficialname(name);
String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);
List<Repository> repos = Converter.jsonToRepositoryList(new JSONObject(rs));
for (Repository r : repos)
updateRepositoryInfo(r);
return repos;
}
@Override
public List<RepositoryInterface> getRepositoryInterface(String id) throws JSONException {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/")
.path("/{id}")
.build().expand(id).encode();
String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
return Converter.jsonToRepositoryInterfaceList(new JSONObject(rs));
}
@Override
public Repository addRepository(String datatype, Repository repository) throws Exception {
LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId());
repository.setCountryCode(countriesMap.get(repository.getCountryName()));
repository.setActivationId(UUID.randomUUID().toString());
repository.setCollectedFrom("infrastruct_::openaire");
if (datatype.equals("journal")) {
repository.setId("openaire____::issn" + repository.getIssn());
repository.setNamespacePrefix("issn" + repository.getIssn());
this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
} else if (datatype.equals("aggregator")) {
repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialName()));
repository.setNamespacePrefix(DigestUtils.md5Hex(repository.getOfficialName()).substring(0, 12));
this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
} else {
this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication());
}
// TODO: move the following code elsewhere (creation and assignment of role to user) ??
// Create new role
String newRoleName = roleMappingService.getRoleIdByRepoId(repository.getId());
Role newRole = new Role(newRoleName, repository.getOfficialName());
Integer couId = null;
try {
couId = registryCalls.createRole(newRole);
} catch (HttpClientErrorException e) {
couId = registryCalls.getCouId(newRoleName);
if (couId == null) {
LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);
}
} catch (Exception e) {
LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);
throw e;
}
// Assign new role to the user that created it
Integer coPersonId = registryCalls.getCoPersonIdByIdentifier();
if (couId != null) {
Integer role = registryCalls.getRoleId(coPersonId, couId);
try {
registryCalls.assignMemberRole(coPersonId, couId, role);
// Add role to current user authorities
authoritiesUpdater.addRole(roleMappingService.convertRepoIdToAuthority(repository.getId()));
} catch (Exception e) {
LOGGER.debug("Exception on assign role to user during add repository", e);
throw e;
}
}
return repository;
}
/* update method acting as add -> send email with registration topic/body*/
private Repository latentUpdate(Repository repository, Authentication authentication) throws Exception {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/update/")
.build()
.encode();
String json_repository = Converter.repositoryObjectToJson(repository);
LOGGER.debug("JSON to add(update) -> " + json_repository);
HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);
ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
try {
emailUtils.sendUserRegistrationEmail(repository, authentication);
emailUtils.sendAdminRegistrationEmail(repository, authentication);
} catch (Exception e) {
LOGGER.error("Error sending email", e);
}
} else
LOGGER.error("Error storing repository: " + responseEntity.getBody().toString());
return repository;
}
@Override
public Repository updateRepository(Repository repository, Authentication authentication) throws Exception {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/update/")
.build()
.encode();
String json_repository = Converter.repositoryObjectToJson(repository);
LOGGER.debug("JSON to update -> " + json_repository);
HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);
ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity
, ResponseEntity.class);
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
try {
emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication);
emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication);
} catch (Exception e) {
LOGGER.error("Error sending emails: " + e);
}
} else
LOGGER.debug(responseEntity.getBody().toString());
return repository;
}
private void storeRepository(Repository repository, Authentication authentication) throws Exception {
Date utilDate = new Date();
Timestamp date = new Timestamp(utilDate.getTime());
repository.setDateOfCollection(date);
repository.setAggregator("OPENAIRE");
repository.setCountryCode(countriesMap.get(repository.getCountryName()));
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/add/")
.build()
.encode();
String json_repository = Converter.repositoryObjectToJson(repository);
HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);
ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
try {
emailUtils.sendUserRegistrationEmail(repository, authentication);
emailUtils.sendAdminRegistrationEmail(repository, authentication);
} catch (Exception e) {
LOGGER.error("Error sending emails: " + e);
}
} else {
LOGGER.debug(responseEntity.getBody().toString());
}
}
@Override
public void deleteRepositoryInterface(String id,
String registeredBy) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/")
.path("/{id}")
.build().expand(id).encode();
LOGGER.debug(uriComponents.toUri());
restTemplate.delete(uriComponents.toUri());
}
@Override
public RepositoryInterface addRepositoryInterface(String datatype,
String repoId,
String registeredBy,
String comment, RepositoryInterface repositoryInterface) throws Exception {
Repository e = this.getRepositoryById(repoId);
repositoryInterface = createRepositoryInterface(e, repositoryInterface, datatype);
String json_interface = Converter.repositoryInterfaceObjectToJson(e, repositoryInterface);
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/add/")
.build()
.encode();
HttpEntity<String> httpEntity = new HttpEntity<>(json_interface, httpHeaders);
restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class);
try {
emailUtils.sendAdminRegisterInterfaceEmail(e, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
emailUtils.sendUserRegisterInterfaceEmail(e, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
} catch (Exception ex) {
LOGGER.error("Error sending emails: " + ex);
}
submitInterfaceValidation(e, registeredBy, repositoryInterface, false);
return repositoryInterface;
}
@Override
public RepositoryInterface updateRepositoryInterface(String repoId,
String registeredBy,
String comment, RepositoryInterface repositoryInterface) throws Exception {
this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseUrl());
this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompliance());
this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet());
Repository repository = this.getRepositoryById(repoId);
try {
try {
emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());
} catch (Exception e) {
LOGGER.error("Error sending emails: " + e);
}
} catch (Exception e) {
LOGGER.warn("Could not send emails", e);
}
submitInterfaceValidation(getRepositoryById(repoId), registeredBy, repositoryInterface, true);
return repositoryInterface;
}
private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting) throws ValidatorServiceException {
JobForValidation job = new JobForValidation();
job.setActivationId(UUID.randomUUID().toString());
job.setAdminEmails(Collections.singletonList(this.adminEmail));
job.setBaseUrl(iFace.getBaseUrl());
job.setDatasourceId(repo.getId());
job.setDesiredCompatibilityLevel(iFace.getDesiredCompatibilityLevel());
job.setInterfaceId(iFace.getId());
job.setOfficialName(repo.getOfficialName());
job.setRepoType(repo.getDatasourceType());
job.setUserEmail(userEmail);
job.setValidationSet((iFace.getAccessSet().isEmpty() ? "none" : iFace.getAccessSet()));
job.setRecords(-1);
job.setRegistration(!updateExisting);
job.setUpdateExisting(updateExisting);
this.validatorService.submitJobForValidation(job);
}
private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) {
iFace.setContentDescription("metadata");
iFace.setCompliance("UNKNOWN");
if (datatype.equals("re3data"))
iFace.setAccessFormat("oai_datacite");
else
iFace.setAccessFormat("oai_dc");
if (repo.getDatasourceClass() != null && !repo.getDatasourceClass().isEmpty())
iFace.setTypology(repo.getDatasourceClass());
else if (datatype.equalsIgnoreCase("journal"))
iFace.setTypology("pubsrepository::journal");
else if (datatype.equalsIgnoreCase("aggregator"))
iFace.setTypology("aggregator::pubsrepository::unknown");
else if (datatype.equalsIgnoreCase("opendoar"))
iFace.setTypology("pubsrepository::unknown");
else if (datatype.equalsIgnoreCase("re3data"))
iFace.setTypology("datarepository::unknown");
iFace.setRemovable(true);
iFace.setAccessProtocol("oai");
iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']");
iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8));
if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) {
LOGGER.debug("set is empty: " + iFace.getAccessSet());
iFace.removeAccessSet();
iFace.setAccessSet("none");
}
return iFace;
}
@Override
public List<String> getDnetCountries() {
LOGGER.debug("Getting dnet-countries!");
return Converter.readFile("countries.txt");
}
@Override
public List<String> getTypologies() {
return Converter.readFile("typologies.txt");
}
@Override
public List<Timezone> getTimezones() {
List<String> timezones = Converter.readFile("timezones.txt");
return Converter.toTimezones(timezones);
}
@Override
public List<String> getUrlsOfUserRepos(String userEmail,
String page,
String size) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/api/baseurl/")
.path("/{page}/{size}")
.build().expand(page, size).encode();
RequestFilter requestFilter = new RequestFilter();
requestFilter.setRegisteredby(userEmail);
return Arrays.asList(restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class));
}
@Override
public List<String> getDatasourceVocabularies(String mode) {
List<String> resultSet = new ArrayList<>();
for (Map.Entry<String, String> entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) {
if (mode.equalsIgnoreCase("aggregator")) {
if (entry.getKey().contains("aggregator"))
resultSet.add(entry.getValue());
} else if (mode.equalsIgnoreCase("journal")) {
if (entry.getKey().contains("journal"))
resultSet.add(entry.getValue());
} else if (mode.equalsIgnoreCase("opendoar")) {
if (entry.getKey().contains("pubsrepository"))
resultSet.add(entry.getValue());
} else if (mode.equalsIgnoreCase("re3data")) {
if (entry.getKey().contains("datarepository"))
resultSet.add(entry.getValue());
}
}
return resultSet;
}
private Vocabulary getVocabulary(String vocName) {
if (!vocabularyMap.containsKey(vocName)) {
vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
}
return vocabularyMap.get(vocName);
}
@Override
public Map<String, String> getCompatibilityClasses(String mode) {
LOGGER.debug("Getting compatibility classes for mode: " + mode);
Map<String, String> retMap = new HashMap<String, String>();
Map<String, String> compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap();
boolean foundData = false;
for (Map.Entry<String, String> entry : compatibilityClasses.entrySet()) {
if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL))
return compatibilityClasses;
else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) {
if (entry.getKey().matches("^openaire[1-9].0_data$")) {
retMap.put(entry.getKey(), entry.getValue());
foundData = true;
}
} else {
if (entry.getKey().matches("^openaire[1-9].0$") || entry.getKey().equals("driver"))
retMap.put(entry.getKey(), entry.getValue());
}
}
//TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED
if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA) && !foundData)
retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)");
return retMap;
}
@Override
public Map<String, String> getDatasourceClasses(String mode) {
LOGGER.debug("Getting datasource classes for mode: " + mode);
Map<String, String> retMap = new HashMap<String, String>();
for (Map.Entry<String, String> entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) {
if (mode.equalsIgnoreCase("aggregator")) {
if (entry.getKey().contains("aggregator"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("journal")) {
if (entry.getKey().contains("journal"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("opendoar")) {
if (entry.getKey().contains("pubsrepository"))
retMap.put(entry.getKey(), entry.getValue());
} else if (mode.equalsIgnoreCase("re3data")) {
if (entry.getKey().contains("datarepository"))
retMap.put(entry.getKey(), entry.getValue());
}
}
return filterResults(retMap, mode);
}
private Map<String, String> filterResults(Map<String, String> map, String mode) {
HashMap<String, String> filteredMap = new HashMap<>();
for (String key : map.keySet())
if (dataSourceClass.get(mode).contains(key))
filteredMap.put(key, map.get(key));
return filteredMap;
}
@Override
public String getCountryName(String countryCode) {
return inverseCountriesMap.get(countryCode);
}
@Override
public MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException {
try {
MetricsInfo metricsInfo = new MetricsInfo();
metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL);
metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId)));
return metricsInfo;
} catch (Exception e) {
LOGGER.error("Error while getting metrics info for repository: ", e);
throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR);
}
}
@Override
public Map<String, String> getListLatestUpdate(String mode) throws JSONException {
if (mode.equals("opendoar"))
return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate());
else
/*
* first api of re3data has null value on collection date
* */
return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::" + mode).get(1).getLastCollectionDate());
}
private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/oaiset")
.queryParam("dsId", repositoryId)
.queryParam("apiId", repositoryInterfaceId)
.queryParam("oaiSet", validationSet)
.build().encode();
restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, null, ResponseEntity.class);
}
private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/baseurl")
.queryParam("dsId", repositoryId)
.queryParam("apiId", repositoryInterfaceId)
.queryParam("baseUrl", baseUrl)
.build().encode();
restTemplate.postForObject(uriComponents.toUri(), null, String.class);
}
private void updateCompliance(String repositoryId, String repositoryInterfaceId, String compliance) {
UriComponents uriComponents = UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/api/compliance")
.queryParam("dsId", repositoryId)
.queryParam("apiId", repositoryInterfaceId)
.queryParam("compliance", compliance)
.build().encode();
restTemplate.postForObject(uriComponents.toUri(), null, String.class);
}
private MetricsNumbers getMetricsNumbers(String openAIREID) throws BrokerException {
//build the uri params
UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks");
//create new template engine
RestTemplate template = new RestTemplate();
template.getMessageConverters().add(new MappingJackson2HttpMessageConverter());
ResponseEntity<MetricsNumbers> resp;
//communicate with endpoint
resp = template.exchange(
builder.build().encode().toUri(),
HttpMethod.GET,
null,
new ParameterizedTypeReference<MetricsNumbers>() {
});
return resp.getBody();
}
private String getOpenAIREId(String repoId) {
if (repoId != null && repoId.contains("::")) {
return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]);
}
return null;
}
private UriComponents searchDatasource(String page, String size) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/search/")
.path("/{page}/{size}/")
.queryParam("requestSortBy", "officialname")
.queryParam("order", "ASCENDING")
.build().expand(page, size).encode();
}
private UriComponents searchSnipperDatasource(String page, String size) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/searchsnippet/")
.path("/{page}/{size}/")
.queryParam("requestSortBy", "officialname")
.queryParam("order", "ASCENDING")
.build().expand(page, size).encode();
}
private UriComponents searchRegisteredDatasource(String requestSortBy, String order, String page, String size) {
return UriComponentsBuilder
.fromHttpUrl(baseAddress + "/ds/searchregistered/")
.path("/{page}/{size}/")
.queryParam("requestSortBy", requestSortBy)
.queryParam("order", order)
.build().expand(page, size).encode();
}
private String getRepositoryType(String typology) {
return invertedDataSourceClass.get(typology);
}
private List<String> getRoleIdsFromUserRoles(String userEmail) {
Integer coPersonId = registryCalls.getCoPersonIdByEmail(userEmail);
JsonArray roles;
ArrayList<String> roleIds = new ArrayList<>();
ArrayList<Integer> couIds = new ArrayList<>();
if (coPersonId != null) {
roles = registryCalls.getRolesWithStatus(coPersonId, AaiRegistryService.RoleStatus.ACTIVE);
for (JsonElement role : roles) {
JsonObject object = role.getAsJsonObject();
if (object.get("CouId") == null) {
continue;
}
couIds.add(object.get("CouId").getAsInt());
}
roleIds.addAll(registryCalls.getCouNames(couIds).values());
}
return roleIds;
}
}