package eu.dnetlib.repo.manager.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import eu.dnetlib.api.functionality.ValidatorServiceException; import eu.dnetlib.domain.enabling.Vocabulary; import eu.dnetlib.domain.functionality.validator.JobForValidation; import eu.dnetlib.repo.manager.domain.*; import eu.dnetlib.repo.manager.domain.dto.User; import eu.dnetlib.repo.manager.exception.RepositoryServiceException; import eu.dnetlib.repo.manager.exception.ResourceNotFoundException; import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService; import eu.dnetlib.repo.manager.service.security.AuthorizationService; import eu.dnetlib.repo.manager.service.security.RoleMappingService; import eu.dnetlib.repo.manager.utils.Converter; import eu.dnetlib.repo.manager.utils.DateUtils; import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader; import org.apache.commons.codec.digest.DigestUtils; import org.mitre.openid.connect.model.OIDCAuthenticationToken; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.*; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import javax.annotation.PostConstruct; import java.sql.Timestamp; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @Service("repositoryService") public class RepositoryServiceImpl implements RepositoryService { private static final Logger logger = LoggerFactory.getLogger(RepositoryServiceImpl.class); private final AuthorizationService authorizationService; private final RoleMappingService roleMappingService; private final AaiRegistryService registryCalls; private final RestTemplate restTemplate; private final ObjectMapper objectMapper; private final VocabularyLoader vocabularyLoader; private final PiWikService piWikService; private final EmailUtils emailUtils; private final ValidatorService validatorService; private final InterfaceComplianceService interfaceComplianceService; @Value("${services.provide.clients.dsm}") private String baseAddress; @Value("${services.provide.adminEmail}") private String adminEmail; @Value("${services.provide.usageStatisticsDiagramsBaseURL}") private String usageStatisticsDiagramsBaseURL; @Value("${services.provide.usageStatisticsNumbersBaseURL}") private String usageStatisticsNumbersBaseURL; private static final Map> dataSourceClass = new HashMap<>(); private static final Map invertedDataSourceClass = new HashMap<>(); private final String[] vocabularyNames = {"dnet:countries", "dnet:eosc_datasource_types", "dnet:compatibilityLevel"}; private final Map vocabularyMap = new ConcurrentHashMap<>(); private final Map countriesMap = new HashMap<>(); private final Map inverseCountriesMap = new HashMap<>(); private HttpHeaders httpHeaders; @Autowired public RepositoryServiceImpl(AuthorizationService authorizationService, RoleMappingService roleMappingService, AaiRegistryService registryCalls, VocabularyLoader vocabularyLoader, RestTemplate restTemplate, ObjectMapper objectMapper, @Lazy EmailUtils emailUtils, @Lazy ValidatorService validatorService, @Lazy PiWikService piWikService, @Lazy InterfaceComplianceService interfaceComplianceService) { this.authorizationService = authorizationService; this.roleMappingService = roleMappingService; this.registryCalls = registryCalls; this.vocabularyLoader = vocabularyLoader; this.piWikService = piWikService; this.emailUtils = emailUtils; this.validatorService = validatorService; this.restTemplate = restTemplate; this.objectMapper = objectMapper; this.interfaceComplianceService = interfaceComplianceService; } @PostConstruct private void init() { logger.debug("Initialization method of repository api! Updated version!"); for (String key : this.getVocabulary("dnet:datasource_typologies").getAsMap().keySet()) { if (key.contains("aggregator")) { dataSourceClass.putIfAbsent("aggregator", new ArrayList<>()); dataSourceClass.get("aggregator").add(key); } else if (key.contains("crissystem")) { dataSourceClass.putIfAbsent("dris", new ArrayList<>()); dataSourceClass.get("dris").add(key); } else if (key.contains("pubsrepository::journal")) { // do not change order --> dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal")); } else if (key.contains("pubsrepository")) { // do not change order <-- dataSourceClass.putIfAbsent("opendoar", new ArrayList<>()); dataSourceClass.get("opendoar").add(key); } else if (key.contains("datarepository")) { dataSourceClass.putIfAbsent("re3data", Collections.singletonList("datarepository::unknown")); } } for (Map.Entry> entry : dataSourceClass.entrySet()) { entry.getValue().forEach(v -> invertedDataSourceClass.put(v, entry.getKey())); } httpHeaders = new HttpHeaders(); httpHeaders.setContentType(MediaType.APPLICATION_JSON); for (String vocName : vocabularyNames) { vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT)); } Country[] countries = getCountries(); for (Country c : countries) { countriesMap.put(c.getName(), c.getCode()); inverseCountriesMap.put(c.getCode(), c.getName()); } } @Override public Country[] getCountries() { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/countries") .build().encode(); return restTemplate.getForObject(uriComponents.toUri(), Country[].class); } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositories(List ids) { return getRepositories(ids, 0, 10); } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositories(List ids, int page, int size) { logger.debug("Retrieving repositories with ids : {}", String.join(", ", ids)); UriComponents uriComponents = searchDatasourceUri(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size))); RequestFilter requestFilter = new RequestFilter(); List repos = new ArrayList<>(); for (String repoId : ids) { requestFilter.setId(repoId); repos.addAll(searchDatasource(uriComponents, requestFilter, Repository.class).getResults()); } for (Repository r : repos) r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId())); return repos; } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositoriesSnippets(List ids) { return getRepositoriesSnippets(ids, 0, 10); } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositoriesSnippets(List ids, int page, int size) { List resultSet = null; List datasourceDetailsList = new ArrayList<>(); // here page should be 0 UriComponents uriComponents = searchDatasourceSnippetUri(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size))); RequestFilter requestFilter = new RequestFilter(); for (String repoId : ids) { requestFilter.setId(repoId); DatasourceResponse rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, DatasourceResponse.class); if (rs == null) { logger.error("The \"DatasourceResponse\" is null!"); } else { datasourceDetailsList.addAll(rs.getDatasourceInfo()); } } try { resultSet = objectMapper.readValue(objectMapper.writeValueAsString(datasourceDetailsList), objectMapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)); if (logger.isDebugEnabled()) { logger.debug("resultSet: {}", objectMapper.writeValueAsString(resultSet)); } resultSet.parallelStream().forEach(repositorySnippet -> { repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId())); }); } catch (JsonProcessingException e) { logger.error("Error deserializing.", e); } return resultSet; } @Override public List getRepositoriesByCountry(String country, String mode, Boolean managed) { logger.debug("Getting repositories by country!"); int page = 0; int size = 100; String filterKey = "UNKNOWN"; if (mode.equalsIgnoreCase("repository")) filterKey = "Repository"; else if (mode.equalsIgnoreCase("cris")) filterKey = "CRIS system"; logger.debug("Country code equals : {} | Filter mode equals : {}", country, filterKey); UriComponents uriComponents = searchDatasourceSnippetUri(String.valueOf(page), String.valueOf(size)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); requestFilter.setEoscDatasourceType(filterKey); return searchDatasource(uriComponents, requestFilter, RepositorySnippet.class).getResults(); } public List searchRegisteredRepositories(String country, String typology, String englishName, String officialName, String requestSortBy, String order, int page, int pageSize) { logger.debug("Searching registered repositories"); UriComponents uriComponents = searchRegisteredDatasourceUri(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); requestFilter.setTypology(typology); requestFilter.setOfficialname(officialName); requestFilter.setEnglishname(englishName); Paging snippets = searchDatasource(uriComponents, requestFilter, RepositorySnippet.class); return snippets != null ? snippets.getResults() : null; // TODO: return paging when ui is compatible } @Override public Integer getTotalRegisteredRepositories() throws NullPointerException { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/countregistered") .queryParam("fromDate", "1900-01-01") .build().encode(); return restTemplate.getForObject(uriComponents.toUri(), Integer.class); } private Repository updateRepositoryInfo(Repository r) { r.setInterfaces(this.getRepositoryInterface(r.getId())); r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId())); return r; } @Override public List getRepositoriesOfUser(String page, String size) { logger.debug("Retrieving repositories of authenticated user : {}", ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail()); Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles()); return getRepositories(new ArrayList<>(repoIds)); } @Override public List getRepositoriesOfUser(String userEmail, String page, String size) { logger.debug("Retrieving repositories of authenticated user : {}", userEmail); Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRolesByEmail(userEmail)); return getRepositories(new ArrayList<>(repoIds)); } @Override public List getRepositoriesSnippetsOfUser(String page, String size) { return getRepositoriesSnippetsOfUser(null, page, size); } @Override public List getRepositoriesSnippetsOfUser(String userEmail, String page, String size) { int from = Integer.parseInt(page) * Integer.parseInt(size); int to = from + Integer.parseInt(size); List repoIds = new ArrayList<>(); if (userEmail != null && !"".equals(userEmail)) { repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRolesByEmail(userEmail))); } else { repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles())); } if (repoIds.size() < from) { return Collections.emptyList(); } else if (repoIds.size() < to) { to = repoIds.size(); } return getRepositoriesSnippets(repoIds.subList(from, to)); // FIXME: returns less results if some repos are not found } @Override public RepositorySnippet getRepositorySnippetById(String id) throws ResourceNotFoundException { logger.debug("Retrieving repositories with id : {}", id); UriComponents uriComponents = searchDatasourceSnippetUri("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); List repositories = searchDatasource(uriComponents, requestFilter, RepositorySnippet.class).getResults(); if (repositories.isEmpty()) throw new ResourceNotFoundException(); return repositories.get(0); } @Override public Repository getRepositoryById(String id) throws ResourceNotFoundException { logger.debug("Retrieving repositories with id : {}", id); UriComponents uriComponents = searchDatasourceUri("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); List datasources = searchDatasource(uriComponents, requestFilter, Repository.class).getResults(); if (datasources.isEmpty()) throw new ResourceNotFoundException(); return updateRepositoryInfo(datasources.get(0)); } @Override public List getRepositoriesByName(String name, String page, String size) { logger.debug("Retrieving repositories with official name : {}", name); UriComponents uriComponents = searchDatasourceUri("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setOfficialname(name); List repos = searchDatasource(uriComponents, requestFilter, Repository.class).getResults(); for (Repository r : repos) updateRepositoryInfo(r); return repos; } @Override public List getRepositoryInterface(String id) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/") .path("/{id}") .build().expand(id).encode(); List rs = searchApi(uriComponents, null, RepositoryInterface.class).getResults(); // TODO STOP FILTERING OUT "sword", "rest" AND FIX UI! List repositoryInterfaces = new ArrayList<>(); for (RepositoryInterface det : rs) { String protocol = det.getProtocol(); if (!protocol.equals("sword") && !protocol.equals("rest") && !protocol.equals("ftp")) { repositoryInterfaces.add(det); } } return repositoryInterfaces; } @Override public Repository addRepository(String datatype, Repository repository) { logger.debug("storing '{}' repository with id: {}", datatype, repository.getId()); repository.setActivationId(UUID.randomUUID().toString()); repository.setCollectedfrom("infrastruct_::openaire"); // Date now = new Date(); // repository.setRegistrationdate(now); // repository.setConsentTermsOfUseDate(now); // repository.setLastConsentTermsOfUseDate(now); if (datatype.equals("journal")) { repository.setEoscDatasourceType("Journal archive"); repository.setId("openaire____::issn" + repository.getIssn()); repository.setNamespaceprefix("issn" + repository.getIssn()); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); } else if (datatype.equals("aggregator")) { repository.setEoscDatasourceType("Aggregator"); repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialname())); repository.setNamespaceprefix(DigestUtils.md5Hex(repository.getOfficialname()).substring(0, 12)); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); } else { if (repository.getTypology().contains("crissystem")) { repository.setEoscDatasourceType("CRIS system"); } else { repository.setEoscDatasourceType("Repository"); } this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication()); } authorizationService.createAndAssignRoleToAuthenticatedUser(repository.getId(), repository.getOfficialname()); return repository; } /* update method acting as add -> send email with registration topic/body*/ private Repository latentUpdate(Repository repository, Authentication authentication) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/update/") .build() .encode(); HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { emailUtils.sendUserRegistrationEmail(repository, authentication); emailUtils.sendAdminRegistrationEmail(repository, authentication); } else { Object responseBody = responseEntity.getBody(); if (responseBody != null) logger.error("Error updating repository: {}", responseBody); } return repository; } @Override public Repository updateRepository(Repository repository, Authentication authentication) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/update/") .build() .encode(); HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication); emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication); } else { Object responseBody = responseEntity.getBody(); if (responseBody != null) logger.error("Error updating repository: {}", responseBody); } return repository; } private void storeRepository(Repository repository, Authentication authentication) { Date utilDate = new Date(); Timestamp date = new Timestamp(utilDate.getTime()); repository.setDateofcollection(date); repository.setAggregator("OPENAIRE"); // repository.setCountryCode(countriesMap.get(repository.getCountryName())); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/add/") .build() .encode(); HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { emailUtils.sendUserRegistrationEmail(repository, authentication); emailUtils.sendAdminRegistrationEmail(repository, authentication); } else { Object responseBody = responseEntity.getBody(); if (responseBody != null) logger.error("Error storing repository: {}", responseBody); } } @Override public void deleteRepositoryInterface(String id, String registeredBy) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/") .path("/{id}") .build().expand(id).encode(); logger.debug("{}", uriComponents.toUri()); restTemplate.delete(uriComponents.toUri()); } @Override public RepositoryInterface addRepositoryInterface(String datatype, String repoId, String comment, RepositoryInterface repositoryInterface, String desiredCompatibilityLevel) throws Exception { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); Repository repo = this.getRepositoryById(repoId); repositoryInterface = fillInterfaceFields(repo, repositoryInterface, datatype); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/add/") .build().encode(); HttpEntity httpEntity = new HttpEntity<>(repositoryInterface, httpHeaders); restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class); // Explicitly update validation set (updating the interface does not allow updating the set value) this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet()); emailUtils.sendAdminRegisterInterfaceEmail(repo, comment, repositoryInterface, desiredCompatibilityLevel, authentication); emailUtils.sendUserRegisterInterfaceEmail(repo, comment, repositoryInterface, desiredCompatibilityLevel, authentication); String prevCompatibilityLevel = repositoryInterface.getCompatibility(); if ( (desiredCompatibilityLevel != null) && ((prevCompatibilityLevel == null) || ! prevCompatibilityLevel.equals(desiredCompatibilityLevel))) { InterfaceComplianceRequest request = new InterfaceComplianceRequest(repoId, repositoryInterface.getId(), desiredCompatibilityLevel); interfaceComplianceService.create(request); } submitInterfaceValidation(repo, getAuthenticatedUser().getEmail(), repositoryInterface, false, desiredCompatibilityLevel); return repositoryInterface; } @Override public RepositoryInterface updateRepositoryInterface(String repoId, String comment, RepositoryInterface repositoryInterface, String desiredCompatibilityLevel) throws ResourceNotFoundException, ValidatorServiceException { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); Repository repository = this.getRepositoryById(repoId); if (repositoryInterface.getId() != null) { RepositoryInterface existing = getRepositoryInterface(repoId).stream().filter(iFace -> iFace.getId().equals(repositoryInterface.getId())).findFirst().orElse(null); if (existing != null && (existing.getBaseurl() == null || "".equals(existing.getBaseurl()))) { this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseurl()); } } this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet()); emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, desiredCompatibilityLevel, authentication); emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, desiredCompatibilityLevel, authentication); if (desiredCompatibilityLevel != null && (repositoryInterface.getCompatibility() == null || !repositoryInterface.getCompatibility().equals(desiredCompatibilityLevel))) { InterfaceComplianceRequest request = new InterfaceComplianceRequest(repoId, repositoryInterface.getId(), desiredCompatibilityLevel); interfaceComplianceService.create(request); } submitInterfaceValidation(getRepositoryById(repoId), getAuthenticatedUser().getEmail(), repositoryInterface, true, desiredCompatibilityLevel); return repositoryInterface; } @Override public void updateInterfaceCompliance(String repositoryId, String repositoryInterfaceId, String compliance) { this.updateCompliance(repositoryId, repositoryInterfaceId, compliance); } private User getAuthenticatedUser() { return User.from(((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo()); } private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting, String desiredCompatibilityLevel) throws ValidatorServiceException { JobForValidation job = new JobForValidation(); job.setActivationId(UUID.randomUUID().toString()); job.setAdminEmails(Collections.singletonList(this.adminEmail)); job.setBaseUrl(iFace.getBaseurl()); job.setDatasourceId(repo.getId()); job.setDesiredCompatibilityLevel(desiredCompatibilityLevel); job.setInterfaceId(iFace.getId()); job.setOfficialName(repo.getOfficialname()); job.setRepoType(repo.getEoscDatasourceType()); job.setUserEmail(userEmail); String accessSet = iFace.getAccessSet(); job.setValidationSet((accessSet.isEmpty() ? "none" : accessSet)); job.setRecords(-1); job.setRegistration(!updateExisting); job.setUpdateExisting(updateExisting); this.validatorService.submitJobForValidation(job); } private RepositoryInterface fillInterfaceFields(Repository repo, RepositoryInterface iFace, String datatype) { iFace.setDatasource(repo.getId()); iFace.setContentdescription("metadata"); logger.warn("Compatibility level: {}", iFace.getCompatibility()); if (iFace.getCompatibility() == null || iFace.getCompatibility().equals("")) { iFace.setCompatibility("UNKNOWN"); } if (datatype.equals("re3data")) iFace.setAccessFormat("oai_datacite"); else iFace.setAccessFormat("oai_dc"); // TODO: is this the correct functionality? if (repo.getEoscDatasourceType() != null && !repo.getEoscDatasourceType().isEmpty()) iFace.setTypology(repo.getEoscDatasourceType()); else if (datatype.equalsIgnoreCase("journal")) iFace.setTypology("pubsrepository::journal"); else if (datatype.equalsIgnoreCase("aggregator")) iFace.setTypology("aggregator::pubsrepository::unknown"); else if (datatype.equalsIgnoreCase("opendoar")) iFace.setTypology("pubsrepository::unknown"); else if (datatype.equalsIgnoreCase("re3data")) iFace.setTypology("datarepository::unknown"); iFace.setRemovable(true); iFace.setProtocol("oai"); iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']"); iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8)); if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) { logger.debug("set is empty: {}", iFace.getAccessSet()); iFace.setAccessSet("none"); } return iFace; } @Override public List getDnetCountries() { logger.debug("Getting dnet-countries!"); return Converter.readFile("countries.txt"); } @Override public List getTypologies() { return Converter.readFile("typologies.txt"); } @Override public List getTimezones() { List timezones = Converter.readFile("timezones.txt"); return Converter.toTimezones(timezones); } @Override public List getUrlsOfUserRepos(String userEmail, String page, String size) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/api/baseurl/") .path("/{page}/{size}") .build().expand(page, size).encode(); RequestFilter requestFilter = new RequestFilter(); requestFilter.setRegisteredby(userEmail); String[] result = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class); return (result != null) ? Arrays.asList(result) : null; } private Vocabulary getVocabulary(String vocName) { if (!vocabularyMap.containsKey(vocName)) { vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT)); } return vocabularyMap.get(vocName); } @Override public Map getCompatibilityClasses(String mode) { logger.debug("Getting compatibility classes for mode: {}", mode); Map retMap = new HashMap<>(); Map compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap(); boolean foundData = false; for (Map.Entry entry : compatibilityClasses.entrySet()) { if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL)) return compatibilityClasses; else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) { if (ValidatorServiceImpl.OPENAIRE_DATA_REGEX.matcher(entry.getKey()).matches()) { retMap.put(entry.getKey(), entry.getValue()); foundData = true; } } else if (mode.equalsIgnoreCase("cris")) { if (entry.getKey().contains("openaire-cris")) { retMap.put(entry.getKey(), entry.getValue()); foundData = true; } } else { if (ValidatorServiceImpl.OPENAIRE_OR_DRIVER_REGEX.matcher(entry.getKey()).matches()) retMap.put(entry.getKey(), entry.getValue()); } } //TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED if ((mode.equalsIgnoreCase("repository") || mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) && !foundData) retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)"); return retMap; } @Override public Map getDatasourceClasses(String mode) { logger.debug("Getting datasource classes for mode: {}", mode); Map retMap = new HashMap<>(); // TODO: refactor (remove?) for (Map.Entry entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) { if (mode.equalsIgnoreCase("aggregator")) { if (entry.getKey().contains("aggregator")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("journal")) { if (entry.getKey().contains("journal")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("opendoar")) { if (entry.getKey().contains("pubsrepository")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("re3data")) { if (entry.getKey().contains("datarepository")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("dris")) { if (entry.getKey().contains("crissystem")) retMap.put(entry.getKey(), entry.getValue()); } if (mode.equalsIgnoreCase("fairsharing")) { retMap.put(entry.getKey(), entry.getValue()); } } if (mode.equals("fairsharing")) { return retMap; } return filterResults(retMap, mode); } private Map filterResults(Map map, String mode) { HashMap filteredMap = new HashMap<>(); if (map != null && mode != null) { for (String key : map.keySet()) if (dataSourceClass.get(mode).contains(key)) filteredMap.put(key, map.get(key)); return filteredMap; } return Collections.emptyMap(); } @Override public String getCountryName(String countryCode) { return inverseCountriesMap.get(countryCode); } @Override public MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException { try { MetricsInfo metricsInfo = new MetricsInfo(); metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL); metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId))); return metricsInfo; } catch (Exception e) { logger.error("Error while getting metrics info for repository: ", e); throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR); } } @Override public Map getListLatestUpdate(String mode) { Map dates = new HashMap<>(); if (mode.equals("repository")) { dates.put("opendoar", DateUtils.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate())); dates.put("fairsharing", DateUtils.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate())); // create re3data last collection date // dates.put("re3data", Converter.toString(getRepositoryInterface("openaire____::re3data").get(1).getLastCollectionDate())); List re3interfaces = getRepositoryInterface("openaire____::re3data"); String re3Date = null; for (RepositoryInterface interf : re3interfaces) { if (interf.getLastCollectionDate() != null) { re3Date = DateUtils.toString(interf.getLastCollectionDate()); break; } } dates.put("re3data", re3Date); return dates; } else if (mode.equals("cris")) return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate())); else if (mode.equals("opendoar")) // TODO: remove this and else clause return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate())); else /* * first api of re3data has null value on collection date * */ return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate())); } private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/oaiset") .queryParam("dsId", repositoryId) .queryParam("apiId", repositoryInterfaceId) .queryParam("oaiSet", validationSet) .build().encode(); restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, null, ResponseEntity.class); } private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/baseurl") .queryParam("dsId", repositoryId) .queryParam("apiId", repositoryInterfaceId) .queryParam("baseUrl", baseUrl) .build().encode(); restTemplate.postForObject(uriComponents.toUri(), null, String.class); } private void updateCompliance(String repositoryId, String repositoryInterfaceId, String compliance) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/compliance") .queryParam("dsId", repositoryId) .queryParam("apiId", repositoryInterfaceId) .queryParam("compliance", compliance) .build().encode(); restTemplate.postForObject(uriComponents.toUri(), null, String.class); } private MetricsNumbers getMetricsNumbers(String openAIREID) { //build the uri params UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks"); //create new template engine RestTemplate template = new RestTemplate(); template.getMessageConverters().add(new MappingJackson2HttpMessageConverter()); ResponseEntity resp; //communicate with endpoint resp = template.exchange( builder.build().encode().toUri(), HttpMethod.GET, null, new ParameterizedTypeReference() { }); return resp.getBody(); } private String getOpenAIREId(String repoId) { if (repoId != null && repoId.contains("::")) { return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]); } return null; } private UriComponents searchDatasourceUri(String page, String size) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchdetails/") .path("/{page}/{size}/") .queryParam("requestSortBy", "officialname") .queryParam("order", "ASCENDING") .build().expand(page, size).encode(); } private UriComponents searchDatasourceSnippetUri(String page, String size) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchsnippet/") .path("/{page}/{size}/") .queryParam("requestSortBy", "officialname") .queryParam("order", "ASCENDING") .build().expand(page, size).encode(); } private UriComponents searchRegisteredDatasourceUri(String requestSortBy, String order, String page, String size) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchregistered/") .path("/{page}/{size}/") .queryParam("requestSortBy", requestSortBy) .queryParam("order", order) .build().expand(page, size).encode(); } private Paging searchDatasource(UriComponents uriComponents, RequestFilter requestFilter, Class clazz) { Paging repositories = new Paging<>(); ResponseEntity rs; rs = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, new HttpEntity<>(requestFilter), Map.class); if (!rs.getStatusCode().is2xxSuccessful()) { logger.error("Api call not successful. Code: {} | Body: {}", rs.getStatusCode(), rs.getBody()); } if (rs.getBody() == null) { logger.error("DSM response is null : [url={}]", uriComponents.toUri()); } else { try { Header header = objectMapper.readValue(objectMapper.writeValueAsString(rs.getBody().get("header")), Header.class); repositories = Paging.of(header, objectMapper.readValue( objectMapper.writeValueAsString(rs.getBody().get("datasourceInfo")), objectMapper.getTypeFactory().constructCollectionType(List.class, clazz))); } catch (JsonProcessingException e) { logger.error("Error in objectMapper", e); } } return repositories; } private Paging searchApi(UriComponents uriComponents, RequestFilter requestFilter, Class clazz) { Paging repositories = new Paging<>(); ResponseEntity rs; rs = restTemplate.exchange(uriComponents.toUri(), HttpMethod.GET, new HttpEntity<>(requestFilter), Map.class); if (!rs.getStatusCode().is2xxSuccessful()) { logger.error("Api call not successful. Code: {} | Body: {}", rs.getStatusCode(), rs.getBody()); } if (rs.getBody() == null) { logger.error("DSM response is null : [url={}]", uriComponents.toUri()); } else { try { Header header = objectMapper.readValue(objectMapper.writeValueAsString(rs.getBody().get("header")), Header.class); repositories = Paging.of(header, objectMapper.readValue( objectMapper.writeValueAsString(rs.getBody().get("api")), objectMapper.getTypeFactory().constructCollectionType(List.class, clazz))); } catch (JsonProcessingException e) { logger.error("Error in objectMapper", e); } } return repositories; } private String getRepositoryType(String typology) { return invertedDataSourceClass.get(typology); } private List getRoleIdsFromUserRoles(String userEmail) { List coPersonId = registryCalls.getCoPersonIdsByEmail(userEmail); JsonArray roles; ArrayList roleIds = new ArrayList<>(); ArrayList couIds = new ArrayList<>(); if (coPersonId != null) { roles = registryCalls.getRolesWithStatus(coPersonId, AaiRegistryService.RoleStatus.ACTIVE); for (JsonElement role : roles) { JsonElement couId = role.getAsJsonObject().get("CouId"); if (couId != null) couIds.add(couId.getAsInt()); } roleIds.addAll(registryCalls.getCouNames(couIds).values()); } return roleIds; } }