987 lines
44 KiB
Java
987 lines
44 KiB
Java
package eu.dnetlib.repo.manager.service;
|
|
|
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.google.gson.JsonArray;
|
|
import com.google.gson.JsonElement;
|
|
import com.google.gson.JsonObject;
|
|
import eu.dnetlib.api.functionality.ValidatorServiceException;
|
|
import eu.dnetlib.domain.enabling.Vocabulary;
|
|
import eu.dnetlib.domain.functionality.validator.JobForValidation;
|
|
import eu.dnetlib.repo.manager.domain.*;
|
|
import eu.dnetlib.repo.manager.domain.dto.User;
|
|
import eu.dnetlib.repo.manager.exception.RepositoryServiceException;
|
|
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
|
|
import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService;
|
|
import eu.dnetlib.repo.manager.service.security.AuthorizationService;
|
|
import eu.dnetlib.repo.manager.service.security.RoleMappingService;
|
|
import eu.dnetlib.repo.manager.utils.Converter;
|
|
import eu.dnetlib.repo.manager.utils.DateUtils;
|
|
import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;
|
|
import org.apache.commons.codec.digest.DigestUtils;
|
|
import org.mitre.openid.connect.model.OIDCAuthenticationToken;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.context.annotation.Lazy;
|
|
import org.springframework.core.ParameterizedTypeReference;
|
|
import org.springframework.http.*;
|
|
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
|
|
import org.springframework.security.core.Authentication;
|
|
import org.springframework.security.core.context.SecurityContextHolder;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.web.client.RestTemplate;
|
|
import org.springframework.web.util.UriComponents;
|
|
import org.springframework.web.util.UriComponentsBuilder;
|
|
|
|
import javax.annotation.PostConstruct;
|
|
import java.sql.Timestamp;
|
|
import java.util.*;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
|
|
@Service("repositoryService")
|
|
public class RepositoryServiceImpl implements RepositoryService {
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(RepositoryServiceImpl.class);
|
|
|
|
private final AuthorizationService authorizationService;
|
|
private final RoleMappingService roleMappingService;
|
|
private final AaiRegistryService registryCalls;
|
|
private final RestTemplate restTemplate;
|
|
private final ObjectMapper objectMapper;
|
|
private final VocabularyLoader vocabularyLoader;
|
|
private final PiWikService piWikService;
|
|
private final EmailUtils emailUtils;
|
|
private final ValidatorService validatorService;
|
|
private final InterfaceComplianceService interfaceComplianceService;
|
|
|
|
@Value("${services.provide.clients.dsm}")
|
|
private String baseAddress;
|
|
|
|
@Value("${services.provide.adminEmail}")
|
|
private String adminEmail;
|
|
|
|
@Value("${services.provide.usageStatisticsDiagramsBaseURL}")
|
|
private String usageStatisticsDiagramsBaseURL;
|
|
|
|
@Value("${services.provide.usageStatisticsNumbersBaseURL}")
|
|
private String usageStatisticsNumbersBaseURL;
|
|
|
|
private static final Map<String, List<String>> dataSourceClass = new HashMap<>();
|
|
private static final Map<String, String> invertedDataSourceClass = new HashMap<>();
|
|
|
|
|
|
private final String[] vocabularyNames = {"dnet:countries", "dnet:eosc_datasource_types", "dnet:compatibilityLevel"};
|
|
private final Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<>();
|
|
private final Map<String, String> countriesMap = new HashMap<>();
|
|
private final Map<String, String> inverseCountriesMap = new HashMap<>();
|
|
private HttpHeaders httpHeaders;
|
|
|
|
@Autowired
|
|
public RepositoryServiceImpl(AuthorizationService authorizationService,
|
|
RoleMappingService roleMappingService,
|
|
AaiRegistryService registryCalls,
|
|
VocabularyLoader vocabularyLoader,
|
|
RestTemplate restTemplate,
|
|
ObjectMapper objectMapper,
|
|
@Lazy EmailUtils emailUtils,
|
|
@Lazy ValidatorService validatorService,
|
|
@Lazy PiWikService piWikService,
|
|
@Lazy InterfaceComplianceService interfaceComplianceService) {
|
|
this.authorizationService = authorizationService;
|
|
this.roleMappingService = roleMappingService;
|
|
this.registryCalls = registryCalls;
|
|
this.vocabularyLoader = vocabularyLoader;
|
|
this.piWikService = piWikService;
|
|
this.emailUtils = emailUtils;
|
|
this.validatorService = validatorService;
|
|
this.restTemplate = restTemplate;
|
|
this.objectMapper = objectMapper;
|
|
this.interfaceComplianceService = interfaceComplianceService;
|
|
}
|
|
|
|
@PostConstruct
|
|
private void init() {
|
|
logger.debug("Initialization method of repository api! Updated version!");
|
|
|
|
for (String key : this.getVocabulary("dnet:datasource_typologies").getAsMap().keySet()) {
|
|
if (key.contains("aggregator")) {
|
|
dataSourceClass.putIfAbsent("aggregator", new ArrayList<>());
|
|
dataSourceClass.get("aggregator").add(key);
|
|
} else if (key.contains("crissystem")) {
|
|
dataSourceClass.putIfAbsent("dris", new ArrayList<>());
|
|
dataSourceClass.get("dris").add(key);
|
|
} else if (key.contains("pubsrepository::journal")) { // do not change order -->
|
|
dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal"));
|
|
} else if (key.contains("pubsrepository")) { // do not change order <--
|
|
dataSourceClass.putIfAbsent("opendoar", new ArrayList<>());
|
|
dataSourceClass.get("opendoar").add(key);
|
|
} else if (key.contains("datarepository")) {
|
|
dataSourceClass.putIfAbsent("re3data", Collections.singletonList("datarepository::unknown"));
|
|
}
|
|
}
|
|
|
|
for (Map.Entry<String, List<String>> entry : dataSourceClass.entrySet()) {
|
|
entry.getValue().forEach(v -> invertedDataSourceClass.put(v, entry.getKey()));
|
|
}
|
|
|
|
httpHeaders = new HttpHeaders();
|
|
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
|
|
|
|
for (String vocName : vocabularyNames) {
|
|
vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
|
|
}
|
|
|
|
Country[] countries = getCountries();
|
|
for (Country c : countries) {
|
|
countriesMap.put(c.getName(), c.getCode());
|
|
inverseCountriesMap.put(c.getCode(), c.getName());
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public Country[] getCountries() {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/countries")
|
|
.build().encode();
|
|
return restTemplate.getForObject(uriComponents.toUri(), Country[].class);
|
|
}
|
|
|
|
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
|
|
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
|
|
// another way for paging must be implemented.
|
|
@Override
|
|
public List<Repository> getRepositories(List<String> ids) {
|
|
return getRepositories(ids, 0, 10);
|
|
}
|
|
|
|
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
|
|
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
|
|
// another way for paging must be implemented.
|
|
@Override
|
|
public List<Repository> getRepositories(List<String> ids, int page, int size) {
|
|
logger.debug("Retrieving repositories with ids : {}", String.join(", ", ids));
|
|
UriComponents uriComponents = searchDatasourceUri(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
|
|
List<Repository> repos = new ArrayList<>();
|
|
|
|
for (String repoId : ids) {
|
|
requestFilter.setId(repoId);
|
|
repos.addAll(searchDatasource(uriComponents, requestFilter, Repository.class).getResults());
|
|
}
|
|
|
|
for (Repository r : repos)
|
|
r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
|
|
return repos;
|
|
}
|
|
|
|
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
|
|
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
|
|
// another way for paging must be implemented.
|
|
@Override
|
|
public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids) {
|
|
return getRepositoriesSnippets(ids, 0, 10);
|
|
}
|
|
|
|
// FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used
|
|
// and the "requestFilter.setId(repoId)" should return only one result at a time, thus,
|
|
// another way for paging must be implemented.
|
|
@Override
|
|
public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids, int page, int size) {
|
|
List<RepositorySnippet> resultSet = null;
|
|
List<DatasourceDetails> datasourceDetailsList = new ArrayList<>();
|
|
|
|
// here page should be 0
|
|
UriComponents uriComponents = searchDatasourceSnippetUri(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
|
|
for (String repoId : ids) {
|
|
requestFilter.setId(repoId);
|
|
|
|
DatasourceResponse rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, DatasourceResponse.class);
|
|
if (rs == null) {
|
|
logger.error("The \"DatasourceResponse\" is null!");
|
|
} else {
|
|
datasourceDetailsList.addAll(rs.getDatasourceInfo());
|
|
}
|
|
}
|
|
|
|
try {
|
|
resultSet = objectMapper.readValue(objectMapper.writeValueAsString(datasourceDetailsList),
|
|
objectMapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class));
|
|
if (logger.isDebugEnabled()) {
|
|
logger.debug("resultSet: {}", objectMapper.writeValueAsString(resultSet));
|
|
}
|
|
resultSet.parallelStream().forEach(repositorySnippet -> {
|
|
repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId()));
|
|
});
|
|
} catch (JsonProcessingException e) {
|
|
logger.error("Error deserializing.", e);
|
|
}
|
|
|
|
return resultSet;
|
|
}
|
|
|
|
|
|
@Override
|
|
public List<RepositorySnippet> getRepositoriesByCountry(String country,
|
|
String mode,
|
|
Boolean managed) {
|
|
logger.debug("Getting repositories by country!");
|
|
int page = 0;
|
|
int size = 100;
|
|
|
|
String filterKey = "UNKNOWN";
|
|
if (mode.equalsIgnoreCase("repository"))
|
|
filterKey = "Repository";
|
|
else if (mode.equalsIgnoreCase("cris"))
|
|
filterKey = "CRIS system";
|
|
|
|
|
|
logger.debug("Country code equals : {} | Filter mode equals : {}", country, filterKey);
|
|
|
|
UriComponents uriComponents = searchDatasourceSnippetUri(String.valueOf(page), String.valueOf(size));
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
requestFilter.setCountry(country);
|
|
requestFilter.setEoscDatasourceType(filterKey);
|
|
|
|
return searchDatasource(uriComponents, requestFilter, RepositorySnippet.class).getResults();
|
|
}
|
|
|
|
public List<RepositorySnippet> searchRegisteredRepositories(String country, String typology, String englishName,
|
|
String officialName, String requestSortBy, String order, int page, int pageSize) {
|
|
|
|
logger.debug("Searching registered repositories");
|
|
|
|
UriComponents uriComponents = searchRegisteredDatasourceUri(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize));
|
|
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
requestFilter.setCountry(country);
|
|
requestFilter.setTypology(typology);
|
|
requestFilter.setOfficialname(officialName);
|
|
requestFilter.setEnglishname(englishName);
|
|
|
|
Paging<RepositorySnippet> snippets = searchDatasource(uriComponents, requestFilter, RepositorySnippet.class);
|
|
return snippets != null ? snippets.getResults() : null; // TODO: return paging when ui is compatible
|
|
}
|
|
|
|
@Override
|
|
public Integer getTotalRegisteredRepositories() throws NullPointerException {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/countregistered")
|
|
.queryParam("fromDate", "1900-01-01")
|
|
.build().encode();
|
|
|
|
return restTemplate.getForObject(uriComponents.toUri(), Integer.class);
|
|
}
|
|
|
|
private Repository updateRepositoryInfo(Repository r) {
|
|
r.setInterfaces(this.getRepositoryInterface(r.getId()));
|
|
r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));
|
|
return r;
|
|
}
|
|
|
|
@Override
|
|
public List<Repository> getRepositoriesOfUser(String page, String size) {
|
|
logger.debug("Retrieving repositories of authenticated user : {}",
|
|
((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail());
|
|
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());
|
|
return getRepositories(new ArrayList<>(repoIds));
|
|
}
|
|
|
|
@Override
|
|
public List<Repository> getRepositoriesOfUser(String userEmail, String page, String size) {
|
|
logger.debug("Retrieving repositories of authenticated user : {}", userEmail);
|
|
Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRolesByEmail(userEmail));
|
|
return getRepositories(new ArrayList<>(repoIds));
|
|
}
|
|
|
|
@Override
|
|
public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String page, String size) {
|
|
return getRepositoriesSnippetsOfUser(null, page, size);
|
|
}
|
|
|
|
@Override
|
|
public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String userEmail, String page, String size) {
|
|
int from = Integer.parseInt(page) * Integer.parseInt(size);
|
|
int to = from + Integer.parseInt(size);
|
|
List<String> repoIds = new ArrayList<>();
|
|
if (userEmail != null && !"".equals(userEmail)) {
|
|
repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRolesByEmail(userEmail)));
|
|
} else {
|
|
repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles()));
|
|
}
|
|
|
|
if (repoIds.size() < from) {
|
|
return Collections.emptyList();
|
|
} else if (repoIds.size() < to) {
|
|
to = repoIds.size();
|
|
}
|
|
return getRepositoriesSnippets(repoIds.subList(from, to)); // FIXME: returns less results if some repos are not found
|
|
}
|
|
|
|
@Override
|
|
public RepositorySnippet getRepositorySnippetById(String id) throws ResourceNotFoundException {
|
|
|
|
logger.debug("Retrieving repositories with id : {}", id);
|
|
UriComponents uriComponents = searchDatasourceSnippetUri("0", "100");
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
requestFilter.setId(id);
|
|
|
|
List<RepositorySnippet> repositories = searchDatasource(uriComponents, requestFilter, RepositorySnippet.class).getResults();
|
|
|
|
if (repositories.isEmpty())
|
|
throw new ResourceNotFoundException();
|
|
|
|
return repositories.get(0);
|
|
}
|
|
|
|
@Override
|
|
public Repository getRepositoryById(String id) throws ResourceNotFoundException {
|
|
|
|
logger.debug("Retrieving repositories with id : {}", id);
|
|
|
|
UriComponents uriComponents = searchDatasourceUri("0", "100");
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
requestFilter.setId(id);
|
|
|
|
List<Repository> datasources = searchDatasource(uriComponents, requestFilter, Repository.class).getResults();
|
|
if (datasources.isEmpty())
|
|
throw new ResourceNotFoundException();
|
|
|
|
return updateRepositoryInfo(datasources.get(0));
|
|
}
|
|
|
|
|
|
@Override
|
|
public List<Repository> getRepositoriesByName(String name,
|
|
String page,
|
|
String size) {
|
|
|
|
logger.debug("Retrieving repositories with official name : {}", name);
|
|
UriComponents uriComponents = searchDatasourceUri("0", "100");
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
requestFilter.setOfficialname(name);
|
|
|
|
List<Repository> repos = searchDatasource(uriComponents, requestFilter, Repository.class).getResults();
|
|
for (Repository r : repos)
|
|
updateRepositoryInfo(r);
|
|
return repos;
|
|
|
|
}
|
|
|
|
@Override
|
|
public List<RepositoryInterface> getRepositoryInterface(String id) {
|
|
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/api/")
|
|
.path("/{id}")
|
|
.build().expand(id).encode();
|
|
|
|
List<RepositoryInterface> rs = searchApi(uriComponents, null, RepositoryInterface.class).getResults();
|
|
|
|
// TODO STOP FILTERING OUT "sword", "rest" AND FIX UI!
|
|
List<RepositoryInterface> repositoryInterfaces = new ArrayList<>();
|
|
for (RepositoryInterface det : rs) {
|
|
String protocol = det.getProtocol();
|
|
if (!protocol.equals("sword") &&
|
|
!protocol.equals("rest") &&
|
|
!protocol.equals("ftp")) {
|
|
repositoryInterfaces.add(det);
|
|
}
|
|
}
|
|
|
|
return repositoryInterfaces;
|
|
}
|
|
|
|
@Override
|
|
public Repository addRepository(String datatype, Repository repository) {
|
|
|
|
logger.debug("storing '{}' repository with id: {}", datatype, repository.getId());
|
|
|
|
repository.setActivationId(UUID.randomUUID().toString());
|
|
repository.setCollectedfrom("infrastruct_::openaire");
|
|
|
|
// Date now = new Date();
|
|
// repository.setRegistrationdate(now);
|
|
// repository.setConsentTermsOfUseDate(now);
|
|
// repository.setLastConsentTermsOfUseDate(now);
|
|
|
|
if (datatype.equals("journal")) {
|
|
repository.setEoscDatasourceType("Journal archive");
|
|
repository.setId("openaire____::issn" + repository.getIssn());
|
|
repository.setNamespaceprefix("issn" + repository.getIssn());
|
|
this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
|
|
} else if (datatype.equals("aggregator")) {
|
|
repository.setEoscDatasourceType("Aggregator");
|
|
repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialname()));
|
|
repository.setNamespaceprefix(DigestUtils.md5Hex(repository.getOfficialname()).substring(0, 12));
|
|
this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());
|
|
} else {
|
|
if (repository.getTypology().contains("crissystem")) {
|
|
repository.setEoscDatasourceType("CRIS system");
|
|
} else {
|
|
repository.setEoscDatasourceType("Repository");
|
|
}
|
|
this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication());
|
|
}
|
|
|
|
authorizationService.createAndAssignRoleToAuthenticatedUser(repository.getId(), repository.getOfficialname());
|
|
|
|
return repository;
|
|
}
|
|
|
|
/* update method acting as add -> send email with registration topic/body*/
|
|
private Repository latentUpdate(Repository repository, Authentication authentication) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/update/")
|
|
.build()
|
|
.encode();
|
|
|
|
HttpEntity<Repository> httpEntity = new HttpEntity<>(repository, httpHeaders);
|
|
ResponseEntity<ResponseEntity> responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
|
|
|
|
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
|
|
emailUtils.sendUserRegistrationEmail(repository, authentication);
|
|
emailUtils.sendAdminRegistrationEmail(repository, authentication);
|
|
} else {
|
|
Object responseBody = responseEntity.getBody();
|
|
if (responseBody != null)
|
|
logger.error("Error updating repository: {}", responseBody);
|
|
}
|
|
|
|
return repository;
|
|
}
|
|
|
|
@Override
|
|
public Repository updateRepository(Repository repository, Authentication authentication) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/update/")
|
|
.build()
|
|
.encode();
|
|
|
|
HttpEntity<Repository> httpEntity = new HttpEntity<>(repository, httpHeaders);
|
|
ResponseEntity<ResponseEntity> responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
|
|
|
|
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
|
|
emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication);
|
|
emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication);
|
|
} else {
|
|
Object responseBody = responseEntity.getBody();
|
|
if (responseBody != null)
|
|
logger.error("Error updating repository: {}", responseBody);
|
|
}
|
|
|
|
return repository;
|
|
}
|
|
|
|
private void storeRepository(Repository repository, Authentication authentication) {
|
|
|
|
Date utilDate = new Date();
|
|
Timestamp date = new Timestamp(utilDate.getTime());
|
|
repository.setDateofcollection(date);
|
|
repository.setAggregator("OPENAIRE");
|
|
// repository.setCountryCode(countriesMap.get(repository.getCountryName()));
|
|
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/add/")
|
|
.build()
|
|
.encode();
|
|
HttpEntity<Repository> httpEntity = new HttpEntity<>(repository, httpHeaders);
|
|
ResponseEntity<ResponseEntity> responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);
|
|
|
|
if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {
|
|
emailUtils.sendUserRegistrationEmail(repository, authentication);
|
|
emailUtils.sendAdminRegistrationEmail(repository, authentication);
|
|
} else {
|
|
Object responseBody = responseEntity.getBody();
|
|
if (responseBody != null)
|
|
logger.error("Error storing repository: {}", responseBody);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void deleteRepositoryInterface(String id,
|
|
String registeredBy) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/api/")
|
|
.path("/{id}")
|
|
.build().expand(id).encode();
|
|
logger.debug("{}", uriComponents.toUri());
|
|
restTemplate.delete(uriComponents.toUri());
|
|
}
|
|
|
|
@Override
|
|
public RepositoryInterface addRepositoryInterface(String datatype,
|
|
String repoId,
|
|
String comment,
|
|
RepositoryInterface repositoryInterface,
|
|
String desiredCompatibilityLevel) throws Exception {
|
|
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
|
|
Repository e = this.getRepositoryById(repoId);
|
|
repositoryInterface = fillInterfaceFields(e, repositoryInterface, datatype);
|
|
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/api/add/")
|
|
.build()
|
|
.encode();
|
|
|
|
HttpEntity<RepositoryInterface> httpEntity = new HttpEntity<>(repositoryInterface, httpHeaders);
|
|
restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class);
|
|
|
|
// Explicitly update validation set (updating the interface does not allow updating the set value)
|
|
this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet());
|
|
|
|
emailUtils.sendAdminRegisterInterfaceEmail(e, comment, repositoryInterface, authentication);
|
|
emailUtils.sendUserRegisterInterfaceEmail(e, comment, repositoryInterface, authentication);
|
|
|
|
if (desiredCompatibilityLevel != null && (repositoryInterface.getCompatibility() == null || !repositoryInterface.getCompatibility().equals(desiredCompatibilityLevel))) {
|
|
InterfaceComplianceRequest request = new InterfaceComplianceRequest(repoId, repositoryInterface.getId(), desiredCompatibilityLevel);
|
|
interfaceComplianceService.create(request);
|
|
}
|
|
|
|
submitInterfaceValidation(e, getAuthenticatedUser().getEmail(), repositoryInterface, false, repositoryInterface.getCompatibility());
|
|
|
|
return repositoryInterface;
|
|
}
|
|
|
|
@Override
|
|
public RepositoryInterface updateRepositoryInterface(String repoId,
|
|
String comment,
|
|
RepositoryInterface repositoryInterface,
|
|
String desiredCompatibilityLevel) throws ResourceNotFoundException, ValidatorServiceException {
|
|
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
|
|
Repository repository = this.getRepositoryById(repoId);
|
|
if (repositoryInterface.getId() != null) {
|
|
RepositoryInterface existing = getRepositoryInterface(repoId).stream().filter(iFace -> iFace.getId().equals(repositoryInterface.getId())).findFirst().orElse(null);
|
|
if (existing != null && (existing.getBaseurl() == null || "".equals(existing.getBaseurl()))) {
|
|
this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseurl());
|
|
}
|
|
}
|
|
|
|
this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet());
|
|
|
|
emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, authentication);
|
|
emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, authentication);
|
|
|
|
if (desiredCompatibilityLevel != null && (repositoryInterface.getCompatibility() == null || !repositoryInterface.getCompatibility().equals(desiredCompatibilityLevel))) {
|
|
InterfaceComplianceRequest request = new InterfaceComplianceRequest(repoId, repositoryInterface.getId(), desiredCompatibilityLevel);
|
|
interfaceComplianceService.create(request);
|
|
}
|
|
|
|
submitInterfaceValidation(getRepositoryById(repoId), getAuthenticatedUser().getEmail(), repositoryInterface, true, desiredCompatibilityLevel);
|
|
|
|
return repositoryInterface;
|
|
}
|
|
|
|
@Override
|
|
public void updateInterfaceCompliance(String repositoryId, String repositoryInterfaceId, String compliance) {
|
|
this.updateCompliance(repositoryId, repositoryInterfaceId, compliance);
|
|
}
|
|
|
|
private User getAuthenticatedUser() {
|
|
return User.from(((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo());
|
|
}
|
|
|
|
private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting, String desiredCompatibilityLevel) throws ValidatorServiceException {
|
|
JobForValidation job = new JobForValidation();
|
|
|
|
job.setActivationId(UUID.randomUUID().toString());
|
|
job.setAdminEmails(Collections.singletonList(this.adminEmail));
|
|
job.setBaseUrl(iFace.getBaseurl());
|
|
job.setDatasourceId(repo.getId());
|
|
job.setDesiredCompatibilityLevel(desiredCompatibilityLevel);
|
|
job.setInterfaceId(iFace.getId());
|
|
job.setOfficialName(repo.getOfficialname());
|
|
job.setRepoType(repo.getEoscDatasourceType());
|
|
job.setUserEmail(userEmail);
|
|
String accessSet = iFace.getAccessSet();
|
|
|
|
job.setValidationSet((accessSet.isEmpty() ? "none" : accessSet));
|
|
job.setRecords(-1);
|
|
job.setRegistration(!updateExisting);
|
|
job.setUpdateExisting(updateExisting);
|
|
|
|
this.validatorService.submitJobForValidation(job);
|
|
}
|
|
|
|
private RepositoryInterface fillInterfaceFields(Repository repo, RepositoryInterface iFace, String datatype) {
|
|
|
|
iFace.setDatasource(repo.getId());
|
|
iFace.setContentdescription("metadata");
|
|
|
|
logger.warn("Compatibility level: {}", iFace.getCompatibility());
|
|
if (iFace.getCompatibility() == null || iFace.getCompatibility().equals("")) {
|
|
iFace.setCompatibility("UNKNOWN");
|
|
}
|
|
|
|
if (datatype.equals("re3data"))
|
|
iFace.setAccessFormat("oai_datacite");
|
|
else
|
|
iFace.setAccessFormat("oai_dc");
|
|
|
|
|
|
// TODO: is this the correct functionality?
|
|
if (repo.getEoscDatasourceType() != null && !repo.getEoscDatasourceType().isEmpty())
|
|
iFace.setTypology(repo.getEoscDatasourceType());
|
|
else if (datatype.equalsIgnoreCase("journal"))
|
|
iFace.setTypology("pubsrepository::journal");
|
|
else if (datatype.equalsIgnoreCase("aggregator"))
|
|
iFace.setTypology("aggregator::pubsrepository::unknown");
|
|
else if (datatype.equalsIgnoreCase("opendoar"))
|
|
iFace.setTypology("pubsrepository::unknown");
|
|
else if (datatype.equalsIgnoreCase("re3data"))
|
|
iFace.setTypology("datarepository::unknown");
|
|
|
|
iFace.setRemovable(true);
|
|
iFace.setProtocol("oai");
|
|
iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']");
|
|
iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8));
|
|
if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) {
|
|
logger.debug("set is empty: {}", iFace.getAccessSet());
|
|
iFace.setAccessSet("none");
|
|
}
|
|
return iFace;
|
|
}
|
|
|
|
@Override
|
|
public List<String> getDnetCountries() {
|
|
logger.debug("Getting dnet-countries!");
|
|
return Converter.readFile("countries.txt");
|
|
}
|
|
|
|
@Override
|
|
public List<String> getTypologies() {
|
|
return Converter.readFile("typologies.txt");
|
|
}
|
|
|
|
@Override
|
|
public List<Timezone> getTimezones() {
|
|
List<String> timezones = Converter.readFile("timezones.txt");
|
|
return Converter.toTimezones(timezones);
|
|
}
|
|
|
|
@Override
|
|
public List<String> getUrlsOfUserRepos(String userEmail,
|
|
String page,
|
|
String size) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/api/baseurl/")
|
|
.path("/{page}/{size}")
|
|
.build().expand(page, size).encode();
|
|
|
|
RequestFilter requestFilter = new RequestFilter();
|
|
requestFilter.setRegisteredby(userEmail);
|
|
Object result = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class);
|
|
return (result != null) ? Collections.singletonList(result.toString()) : null;
|
|
}
|
|
|
|
private Vocabulary getVocabulary(String vocName) {
|
|
|
|
if (!vocabularyMap.containsKey(vocName)) {
|
|
vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));
|
|
}
|
|
return vocabularyMap.get(vocName);
|
|
}
|
|
|
|
|
|
@Override
|
|
public Map<String, String> getCompatibilityClasses(String mode) {
|
|
|
|
logger.debug("Getting compatibility classes for mode: {}", mode);
|
|
Map<String, String> retMap = new HashMap<>();
|
|
|
|
Map<String, String> compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap();
|
|
boolean foundData = false;
|
|
for (Map.Entry<String, String> entry : compatibilityClasses.entrySet()) {
|
|
if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL))
|
|
return compatibilityClasses;
|
|
else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) {
|
|
if (ValidatorServiceImpl.OPENAIRE_DATA_REGEX.matcher(entry.getKey()).matches()) {
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
foundData = true;
|
|
}
|
|
} else if (mode.equalsIgnoreCase("cris")) {
|
|
if (entry.getKey().contains("openaire-cris")) {
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
foundData = true;
|
|
}
|
|
} else {
|
|
if (ValidatorServiceImpl.OPENAIRE_OR_DRIVER_REGEX.matcher(entry.getKey()).matches())
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
}
|
|
}
|
|
|
|
//TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED
|
|
if ((mode.equalsIgnoreCase("repository") || mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) && !foundData)
|
|
retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)");
|
|
|
|
return retMap;
|
|
}
|
|
|
|
@Override
|
|
public Map<String, String> getDatasourceClasses(String mode) {
|
|
|
|
logger.debug("Getting datasource classes for mode: {}", mode);
|
|
|
|
Map<String, String> retMap = new HashMap<>();
|
|
|
|
// TODO: refactor (remove?)
|
|
for (Map.Entry<String, String> entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) {
|
|
if (mode.equalsIgnoreCase("aggregator")) {
|
|
if (entry.getKey().contains("aggregator"))
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
} else if (mode.equalsIgnoreCase("journal")) {
|
|
if (entry.getKey().contains("journal"))
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
} else if (mode.equalsIgnoreCase("opendoar")) {
|
|
if (entry.getKey().contains("pubsrepository"))
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
} else if (mode.equalsIgnoreCase("re3data")) {
|
|
if (entry.getKey().contains("datarepository"))
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
} else if (mode.equalsIgnoreCase("dris")) {
|
|
if (entry.getKey().contains("crissystem"))
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
}
|
|
if (mode.equalsIgnoreCase("fairsharing")) {
|
|
retMap.put(entry.getKey(), entry.getValue());
|
|
}
|
|
}
|
|
if (mode.equals("fairsharing")) {
|
|
return retMap;
|
|
}
|
|
return filterResults(retMap, mode);
|
|
|
|
}
|
|
|
|
private Map<String, String> filterResults(Map<String, String> map, String mode) {
|
|
|
|
HashMap<String, String> filteredMap = new HashMap<>();
|
|
if (map != null && mode != null) {
|
|
for (String key : map.keySet())
|
|
if (dataSourceClass.get(mode).contains(key))
|
|
filteredMap.put(key, map.get(key));
|
|
|
|
return filteredMap;
|
|
}
|
|
return Collections.emptyMap();
|
|
}
|
|
|
|
@Override
|
|
public String getCountryName(String countryCode) {
|
|
return inverseCountriesMap.get(countryCode);
|
|
}
|
|
|
|
@Override
|
|
public MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException {
|
|
try {
|
|
|
|
MetricsInfo metricsInfo = new MetricsInfo();
|
|
metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL);
|
|
metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId)));
|
|
return metricsInfo;
|
|
|
|
} catch (Exception e) {
|
|
logger.error("Error while getting metrics info for repository: ", e);
|
|
throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public Map<String, String> getListLatestUpdate(String mode) {
|
|
Map<String, String> dates = new HashMap<>();
|
|
if (mode.equals("repository")) {
|
|
dates.put("opendoar", DateUtils.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate()));
|
|
dates.put("fairsharing", DateUtils.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate()));
|
|
// create re3data last collection date
|
|
// dates.put("re3data", Converter.toString(getRepositoryInterface("openaire____::re3data").get(1).getLastCollectionDate()));
|
|
List<RepositoryInterface> re3interfaces = getRepositoryInterface("openaire____::re3data");
|
|
String re3Date = null;
|
|
for (RepositoryInterface interf : re3interfaces) {
|
|
if (interf.getLastCollectionDate() != null) {
|
|
re3Date = DateUtils.toString(interf.getLastCollectionDate());
|
|
break;
|
|
}
|
|
}
|
|
dates.put("re3data", re3Date);
|
|
|
|
return dates;
|
|
} else if (mode.equals("cris"))
|
|
return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate()));
|
|
else if (mode.equals("opendoar")) // TODO: remove this and else clause
|
|
return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
|
|
else
|
|
/*
|
|
* first api of re3data has null value on collection date
|
|
* */
|
|
return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
|
|
}
|
|
|
|
private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/api/oaiset")
|
|
.queryParam("dsId", repositoryId)
|
|
.queryParam("apiId", repositoryInterfaceId)
|
|
.queryParam("oaiSet", validationSet)
|
|
.build().encode();
|
|
restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, null, ResponseEntity.class);
|
|
|
|
}
|
|
|
|
|
|
private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/api/baseurl")
|
|
.queryParam("dsId", repositoryId)
|
|
.queryParam("apiId", repositoryInterfaceId)
|
|
.queryParam("baseUrl", baseUrl)
|
|
.build().encode();
|
|
restTemplate.postForObject(uriComponents.toUri(), null, String.class);
|
|
}
|
|
|
|
private void updateCompliance(String repositoryId, String repositoryInterfaceId, String compliance) {
|
|
UriComponents uriComponents = UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/api/compliance")
|
|
.queryParam("dsId", repositoryId)
|
|
.queryParam("apiId", repositoryInterfaceId)
|
|
.queryParam("compliance", compliance)
|
|
.build().encode();
|
|
restTemplate.postForObject(uriComponents.toUri(), null, String.class);
|
|
}
|
|
|
|
private MetricsNumbers getMetricsNumbers(String openAIREID) {
|
|
|
|
//build the uri params
|
|
UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks");
|
|
|
|
//create new template engine
|
|
RestTemplate template = new RestTemplate();
|
|
template.getMessageConverters().add(new MappingJackson2HttpMessageConverter());
|
|
ResponseEntity<MetricsNumbers> resp;
|
|
|
|
//communicate with endpoint
|
|
resp = template.exchange(
|
|
builder.build().encode().toUri(),
|
|
HttpMethod.GET,
|
|
null,
|
|
new ParameterizedTypeReference<MetricsNumbers>() {
|
|
});
|
|
|
|
return resp.getBody();
|
|
}
|
|
|
|
private String getOpenAIREId(String repoId) {
|
|
|
|
if (repoId != null && repoId.contains("::")) {
|
|
return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
|
|
private UriComponents searchDatasourceUri(String page, String size) {
|
|
|
|
return UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/searchdetails/")
|
|
.path("/{page}/{size}/")
|
|
.queryParam("requestSortBy", "officialname")
|
|
.queryParam("order", "ASCENDING")
|
|
.build().expand(page, size).encode();
|
|
}
|
|
|
|
private UriComponents searchDatasourceSnippetUri(String page, String size) {
|
|
|
|
return UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/searchsnippet/")
|
|
.path("/{page}/{size}/")
|
|
.queryParam("requestSortBy", "officialname")
|
|
.queryParam("order", "ASCENDING")
|
|
.build().expand(page, size).encode();
|
|
}
|
|
|
|
private UriComponents searchRegisteredDatasourceUri(String requestSortBy, String order, String page, String size) {
|
|
|
|
return UriComponentsBuilder
|
|
.fromHttpUrl(baseAddress + "/ds/searchregistered/")
|
|
.path("/{page}/{size}/")
|
|
.queryParam("requestSortBy", requestSortBy)
|
|
.queryParam("order", order)
|
|
.build().expand(page, size).encode();
|
|
}
|
|
|
|
private <T> Paging<T> searchDatasource(UriComponents uriComponents, RequestFilter requestFilter, Class<T> clazz) {
|
|
Paging<T> repositories = new Paging<>();
|
|
ResponseEntity<Map> rs;
|
|
rs = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, new HttpEntity<>(requestFilter), Map.class);
|
|
if (!rs.getStatusCode().is2xxSuccessful()) {
|
|
logger.error("Api call not successful. Code: {} | Body: {}", rs.getStatusCode(), rs.getBody());
|
|
}
|
|
if (rs.getBody() == null) {
|
|
logger.error("DSM response is null : [url={}]", uriComponents.toUri());
|
|
} else {
|
|
try {
|
|
Header header = objectMapper.readValue(objectMapper.writeValueAsString(rs.getBody().get("header")), Header.class);
|
|
repositories = Paging.of(header,
|
|
objectMapper.readValue(
|
|
objectMapper.writeValueAsString(rs.getBody().get("datasourceInfo")),
|
|
objectMapper.getTypeFactory().constructCollectionType(List.class, clazz)));
|
|
} catch (JsonProcessingException e) {
|
|
logger.error("Error in objectMapper", e);
|
|
}
|
|
}
|
|
return repositories;
|
|
}
|
|
|
|
private <T> Paging<T> searchApi(UriComponents uriComponents, RequestFilter requestFilter, Class<T> clazz) {
|
|
Paging<T> repositories = new Paging<>();
|
|
ResponseEntity<Map> rs;
|
|
rs = restTemplate.exchange(uriComponents.toUri(), HttpMethod.GET, new HttpEntity<>(requestFilter), Map.class);
|
|
if (!rs.getStatusCode().is2xxSuccessful()) {
|
|
logger.error("Api call not successful. Code: {} | Body: {}", rs.getStatusCode(), rs.getBody());
|
|
}
|
|
if (rs.getBody() == null) {
|
|
logger.error("DSM response is null : [url={}]", uriComponents.toUri());
|
|
} else {
|
|
try {
|
|
Header header = objectMapper.readValue(objectMapper.writeValueAsString(rs.getBody().get("header")), Header.class);
|
|
repositories = Paging.of(header,
|
|
objectMapper.readValue(
|
|
objectMapper.writeValueAsString(rs.getBody().get("api")),
|
|
objectMapper.getTypeFactory().constructCollectionType(List.class, clazz)));
|
|
} catch (JsonProcessingException e) {
|
|
logger.error("Error in objectMapper", e);
|
|
}
|
|
}
|
|
return repositories;
|
|
}
|
|
|
|
private String getRepositoryType(String typology) {
|
|
return invertedDataSourceClass.get(typology);
|
|
}
|
|
|
|
private List<String> getRoleIdsFromUserRoles(String userEmail) {
|
|
List<Integer> coPersonId = registryCalls.getCoPersonIdsByEmail(userEmail);
|
|
JsonArray roles;
|
|
ArrayList<String> roleIds = new ArrayList<>();
|
|
ArrayList<Integer> couIds = new ArrayList<>();
|
|
if (coPersonId != null) {
|
|
roles = registryCalls.getRolesWithStatus(coPersonId, AaiRegistryService.RoleStatus.ACTIVE);
|
|
for (JsonElement role : roles) {
|
|
JsonObject object = role.getAsJsonObject();
|
|
if (object.get("CouId") == null) {
|
|
continue;
|
|
}
|
|
couIds.add(object.get("CouId").getAsInt());
|
|
}
|
|
|
|
roleIds.addAll(registryCalls.getCouNames(couIds).values());
|
|
|
|
}
|
|
return roleIds;
|
|
}
|
|
}
|