package eu.dnetlib.repo.manager.service; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import eu.dnetlib.api.functionality.ValidatorServiceException; import eu.dnetlib.repo.manager.domain.Repository; import eu.dnetlib.repo.manager.domain.RepositoryInterface; import eu.dnetlib.domain.enabling.Vocabulary; import eu.dnetlib.domain.functionality.validator.JobForValidation; import eu.dnetlib.repo.manager.domain.*; import eu.dnetlib.repo.manager.domain.dto.Role; import eu.dnetlib.repo.manager.domain.dto.User; import eu.dnetlib.repo.manager.exception.BrokerException; import eu.dnetlib.repo.manager.exception.RepositoryServiceException; import eu.dnetlib.repo.manager.exception.ResourceNotFoundException; import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService; import eu.dnetlib.repo.manager.service.security.AuthoritiesUpdater; import eu.dnetlib.repo.manager.service.security.AuthorizationService; import eu.dnetlib.repo.manager.service.security.RoleMappingService; import eu.dnetlib.repo.manager.utils.Converter; import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader; import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.mitre.openid.connect.model.OIDCAuthenticationToken; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.*; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import javax.annotation.PostConstruct; import java.io.IOException; import java.sql.Timestamp; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @Service("repositoryService") public class RepositoryServiceImpl implements RepositoryService { private static final Logger LOGGER = Logger.getLogger(RepositoryServiceImpl.class); private final AuthorizationService authorizationService; private final RoleMappingService roleMappingService; private final AaiRegistryService registryCalls; private final AuthoritiesUpdater authoritiesUpdater; private final RestTemplate restTemplate; private final VocabularyLoader vocabularyLoader; private final PiWikService piWikService; private final EmailUtils emailUtils; private final ValidatorService validatorService; @Value("${services.provide.clients.dsm}") private String baseAddress; @Value("${services.provide.adminEmail}") private String adminEmail; @Value("${services.provide.usageStatisticsDiagramsBaseURL}") private String usageStatisticsDiagramsBaseURL; @Value("${services.provide.usageStatisticsNumbersBaseURL}") private String usageStatisticsNumbersBaseURL; private final Converter converter; private static final Map> dataSourceClass = new HashMap<>(); private static final Map invertedDataSourceClass = new HashMap<>(); private final String[] vocabularyNames = {"dnet:countries", "dnet:eosc_datasource_types", "dnet:compatibilityLevel"}; private final Map vocabularyMap = new ConcurrentHashMap<>(); private final Map countriesMap = new HashMap<>(); private final Map inverseCountriesMap = new HashMap<>(); private HttpHeaders httpHeaders; @Autowired public RepositoryServiceImpl(AuthorizationService authorizationService, RoleMappingService roleMappingService, AaiRegistryService registryCalls, AuthoritiesUpdater authoritiesUpdater, VocabularyLoader vocabularyLoader, RestTemplate restTemplate, Converter converter, @Lazy EmailUtils emailUtils, @Lazy ValidatorService validatorService, @Lazy PiWikService piWikService) { this.authorizationService = authorizationService; this.roleMappingService = roleMappingService; this.registryCalls = registryCalls; this.authoritiesUpdater = authoritiesUpdater; this.vocabularyLoader = vocabularyLoader; this.piWikService = piWikService; this.converter = converter; this.emailUtils = emailUtils; this.validatorService = validatorService; this.restTemplate = restTemplate; } @PostConstruct private void init() { LOGGER.debug("Initialization method of repository api!"); LOGGER.debug("Updated version!"); for (String key : this.getVocabulary("dnet:datasource_typologies").getAsMap().keySet()) { if (key.contains("aggregator")) { dataSourceClass.putIfAbsent("aggregator", new ArrayList<>()); dataSourceClass.get("aggregator").add(key); } else if (key.contains("crissystem")) { dataSourceClass.putIfAbsent("dris", new ArrayList<>()); dataSourceClass.get("dris").add(key); } else if (key.contains("pubsrepository::journal")) { // do not change order --> dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal")); } else if (key.contains("pubsrepository")) { // do not change order <-- dataSourceClass.putIfAbsent("opendoar", new ArrayList<>()); dataSourceClass.get("opendoar").add(key); } else if (key.contains("datarepository")) { dataSourceClass.putIfAbsent("re3data", Collections.singletonList("datarepository::unknown")); } } for (Map.Entry> entry : dataSourceClass.entrySet()) { entry.getValue().forEach(v -> invertedDataSourceClass.put(v, entry.getKey())); } httpHeaders = new HttpHeaders(); httpHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); for (String vocName : vocabularyNames) { vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT)); } Country[] countries = getCountries(); for (Country c : countries) { countriesMap.put(c.getName(), c.getCode()); inverseCountriesMap.put(c.getCode(), c.getName()); } } @Override public Country[] getCountries() { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/countries") .build().encode(); return restTemplate.getForObject(uriComponents.toUri(), Country[].class); } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositories(List ids) throws JSONException { return getRepositories(ids, 0, 10); } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositories(List ids, int page, int size) throws JSONException { List repos = new ArrayList<>(); LOGGER.debug("Retrieving repositories with ids : " + String.join(", ", ids)); UriComponents uriComponents = searchDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size))); RequestFilter requestFilter = new RequestFilter(); for (String repoId : ids) { requestFilter.setId(repoId); List rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, List.class); // repos.addAll(converter.toRepositoryList(new JSONObject(rs))); } for (Repository r : repos) r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId())); return repos; } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositoriesSnippets(List ids) throws Exception { return getRepositoriesSnippets(ids, 0, 10); } // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used // and the "requestFilter.setId(repoId)" should return only one result at a time, thus, // another way for paging must be implemented. @Override public List getRepositoriesSnippets(List ids, int page, int size) throws Exception { List resultSet = new ArrayList<>(); ObjectMapper mapper = new ObjectMapper(); // here page should be 0 UriComponents uriComponents = searchSnipperDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size))); RequestFilter requestFilter = new RequestFilter(); try { for (String repoId : ids) { requestFilter.setId(repoId); String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class))); } } catch (Exception e) { LOGGER.debug("Exception on getRepositoriesSnippetOfUser", e); throw e; } LOGGER.debug("resultSet:" + resultSet); resultSet.parallelStream().forEach(repositorySnippet -> { repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId())); }); return resultSet; } @Override public List getRepositoriesByCountry(String country, String mode, Boolean managed) throws JSONException, IOException { LOGGER.debug("Getting repositories by country!"); int page = 0; int size = 100; List resultSet = new ArrayList<>(); ObjectMapper mapper = new ObjectMapper(); String filterKey = "UNKNOWN"; if (mode.equalsIgnoreCase("repository")) filterKey = "Repository"; else if (mode.equalsIgnoreCase("cris")) filterKey = "CRIS system"; LOGGER.debug("Country code equals : " + country); LOGGER.debug("Filter mode equals : " + filterKey); UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); requestFilter.setEoscDatasourceType(filterKey); String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); while (jsonArray.length() > 0) { resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class))); page += 1; uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size)); rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); } return resultSet; } public List searchRegisteredRepositories(String country, String typology, String englishName, String officialName, String requestSortBy, String order, int page, int pageSize) throws Exception { LOGGER.debug("Searching registered repositories"); Paging snippets = null; ObjectMapper mapper = new ObjectMapper(); UriComponents uriComponents = searchRegisteredDatasource(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); requestFilter.setTypology(typology); requestFilter.setOfficialname(officialName); requestFilter.setEnglishname(englishName); try { String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); if (rs == null) { LOGGER.error(String.format("DSM response is null : [url=%s]", uriComponents.toUri())); } else { JSONObject response = new JSONObject(rs); JSONArray jsonArray = (JSONArray) response.get("datasourceInfo"); Header header = mapper.readValue(response.get("header").toString(), Header.class); snippets = Paging.of(header, mapper.readValue( String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class))); } } catch (Exception e) { LOGGER.error("Error searching registered datasources", e); throw e; } return snippets != null ? snippets.getResults() : null; // TODO: return paging when ui is compatible } @Override public int getTotalRegisteredRepositories() { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/countregistered") .queryParam("fromDate", "1900-01-01") .build().encode(); return restTemplate.getForObject(uriComponents.toUri(), Integer.class); } private Repository updateRepositoryInfo(Repository r) throws JSONException { r.setInterfaces(this.getRepositoryInterface(r.getId())); r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId())); return r; } @Override public List getRepositoriesOfUser(String page, String size) throws JSONException { String userEmail = ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail(); LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail); Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles()); return getRepositories(new ArrayList<>(repoIds)); } @Override public List getRepositoriesOfUser(String userEmail, String page, String size) throws JSONException { LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail); Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail)); return getRepositories(new ArrayList<>(repoIds)); } @Override public List getRepositoriesSnippetsOfUser(String page, String size) throws Exception { return getRepositoriesSnippetsOfUser(null, page, size); } @Override public List getRepositoriesSnippetsOfUser(String userEmail, String page, String size) throws Exception { int from = Integer.parseInt(page) * Integer.parseInt(size); int to = from + Integer.parseInt(size); List repoIds = new ArrayList<>(); if (userEmail != null && !"".equals(userEmail)) { repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail))); } else { repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles())); } if (repoIds.size() < from) { return Collections.emptyList(); } else if (repoIds.size() < to) { to = repoIds.size(); } return getRepositoriesSnippets(repoIds.subList(from, to)); // FIXME: returns less results if some repos are not found } @Override public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException { LOGGER.debug("Retrieving repositories with id : " + id); RepositorySnippet repo; UriComponents uriComponents = searchSnipperDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); if (jsonArray.length() == 0) throw new ResourceNotFoundException(); repo = converter.toRepositorySnippet(jsonArray.getJSONObject(0)); return repo; } @Override public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException { LOGGER.debug("Retrieving repositories with id : " + id); Repository repo; UriComponents uriComponents = searchDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); // String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); // JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); DatasourceResponse response; response = (DatasourceResponse) restTemplate.postForObject(uriComponents.toUri(), requestFilter, DatasourceResponse.class); List datasources = response.getDatasourceInfo(); if (datasources.size() == 0) throw new ResourceNotFoundException(); // repo = converter.toRepository(jsonArray.getJSONObject(0)); // return updateRepositoryInfo(repo); return updateRepositoryInfo(converter.toRepository(datasources.get(0))); } @Override public List getRepositoryAggregations(String id) throws JSONException { LOGGER.debug("Retrieving aggregations for repository with id : " + id); UriComponents uriComponents = getAggregationHistory(id); String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); JSONArray aggregationInfo = new JSONObject(rs).getJSONArray("aggregationInfo"); List aggregationHistory = new ArrayList<>(converter.toAggregationHistory(aggregationInfo)); return aggregationHistory; } @Override public List getRepositoryAggregations(String id, int from, int size) throws JSONException { List res = getRepositoryAggregations(id); return res.subList(from, Math.min(from + size, res.size())); } @Override public Map> getRepositoryAggregationsByYear(String id) throws JSONException { LOGGER.debug("Retrieving aggregations (by year) for repository with id : " + id); List aggregationHistory = getRepositoryAggregations(id); Map> aggregationByYear = new HashMap<>(); return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory); } private Map> createYearMap(List aggregationHistory) { aggregationHistory = aggregationHistory.stream() .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) .collect(Collectors.toList()); return aggregationHistory.stream() .collect(Collectors.groupingBy(AggregationDetails::getYear)); } @Override public List getRepositoriesByName(String name, String page, String size) throws JSONException { LOGGER.debug("Retrieving repositories with official name : " + name); UriComponents uriComponents = searchDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setOfficialname(name); String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); List repos = converter.toRepositoryList(new JSONObject(rs)); for (Repository r : repos) updateRepositoryInfo(r); return repos; } @Override public List getRepositoryInterface(String id) throws JSONException { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/") .path("/{id}") .build().expand(id).encode(); // String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); ApiDetailsResponse rs = restTemplate.getForObject(uriComponents.toUri(), ApiDetailsResponse.class); // TODO STOP FILTERING OUT "sword", "rest" AND FIX UI! List res = new ArrayList<>(); for (ApiDetails det: rs.getApi()) { if (!det.getProtocol().equals("sword") && !det.getProtocol().equals("rest") && !det.getProtocol().equals("ftp")) { res.add(det); } } return converter.toRepositoryInterfaceList(res); } @Override public Repository addRepository(String datatype, Repository repository) throws Exception { LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId()); repository.setActivationId(UUID.randomUUID().toString()); repository.setCollectedfrom("infrastruct_::openaire"); // Date now = new Date(); // repository.setRegistrationdate(now); // repository.setConsentTermsOfUseDate(now); // repository.setLastConsentTermsOfUseDate(now); if (datatype.equals("journal")) { repository.setEoscDatasourceType("Journal archive"); repository.setId("openaire____::issn" + repository.getIssn()); repository.setNamespaceprefix("issn" + repository.getIssn()); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); } else if (datatype.equals("aggregator")) { repository.setEoscDatasourceType("Aggregator"); repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialname())); repository.setNamespaceprefix(DigestUtils.md5Hex(repository.getOfficialname()).substring(0, 12)); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); } else { if (repository.getTypology().contains("crissystem")) { repository.setEoscDatasourceType("CRIS system"); } else { repository.setEoscDatasourceType("Repository"); } this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication()); } // TODO: move the following code elsewhere (creation and assignment of role to user) ?? // Create new role String newRoleName = roleMappingService.getRoleIdByRepoId(repository.getId()); Role newRole = new Role(newRoleName, repository.getOfficialname()); Integer couId = null; try { couId = registryCalls.createRole(newRole); } catch (HttpClientErrorException e) { couId = registryCalls.getCouId(newRoleName); if (couId == null) { LOGGER.error(String.format("Could not create role '%s'", newRoleName), e); } } catch (Exception e) { LOGGER.error(String.format("Could not create role '%s'", newRoleName), e); throw e; } // Assign new role to the user that created it Integer coPersonId = registryCalls.getCoPersonIdByIdentifier(); if (couId != null) { Integer role = registryCalls.getRoleId(coPersonId, couId); try { registryCalls.assignMemberRole(coPersonId, couId, role); // Add role to current user authorities authoritiesUpdater.addRole(roleMappingService.convertRepoIdToAuthority(repository.getId())); } catch (Exception e) { LOGGER.debug("Exception on assign role to user during add repository", e); throw e; } } return repository; } /* update method acting as add -> send email with registration topic/body*/ private Repository latentUpdate(Repository repository, Authentication authentication) throws Exception { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/update/") .build() .encode(); // FIXME: problematic // String json_repository = converter.toJson(repository); // LOGGER.debug("JSON to add(update) -> " + json_repository); HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); // TODO: check if it works (Repository contains extra fields) ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { try { emailUtils.sendUserRegistrationEmail(repository, authentication); emailUtils.sendAdminRegistrationEmail(repository, authentication); } catch (Exception e) { LOGGER.error("Error sending email", e); } } else LOGGER.error("Error storing repository: " + responseEntity.getBody().toString()); return repository; } @Override public Repository updateRepository(Repository repository, Authentication authentication) throws Exception { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/update/") .build() .encode(); // FIXME: problematic // String json_repository = converter.toJson(repository); // LOGGER.debug("JSON to update -> " + json_repository); HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity , ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { try { emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication); emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication); } catch (Exception e) { LOGGER.error("Error sending emails: " + e); } } else LOGGER.debug(responseEntity.getBody().toString()); return repository; } private void storeRepository(Repository repository, Authentication authentication) throws Exception { Date utilDate = new Date(); Timestamp date = new Timestamp(utilDate.getTime()); repository.setDateofcollection(date); repository.setAggregator("OPENAIRE"); // repository.setCountryCode(countriesMap.get(repository.getCountryName())); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/add/") .build() .encode(); // String json_repository = converter.toJson(repository); HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { try { emailUtils.sendUserRegistrationEmail(repository, authentication); emailUtils.sendAdminRegistrationEmail(repository, authentication); } catch (Exception e) { LOGGER.error("Error sending emails: " + e); } } else { LOGGER.debug(responseEntity.getBody().toString()); } } @Override public void deleteRepositoryInterface(String id, String registeredBy) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/") .path("/{id}") .build().expand(id).encode(); LOGGER.debug(uriComponents.toUri()); restTemplate.delete(uriComponents.toUri()); } @Override public RepositoryInterface addRepositoryInterface(String datatype, String repoId, String comment, RepositoryInterface repositoryInterface) throws Exception { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); Repository e = this.getRepositoryById(repoId); repositoryInterface = createRepositoryInterface(e, repositoryInterface, datatype); // String json_interface = converter.toJson(e, repositoryInterface); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/add/") .build() .encode(); HttpEntity httpEntity = new HttpEntity<>(repositoryInterface, httpHeaders); restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class); try { emailUtils.sendAdminRegisterInterfaceEmail(e, comment, repositoryInterface, authentication); emailUtils.sendUserRegisterInterfaceEmail(e, comment, repositoryInterface, authentication); } catch (Exception ex) { LOGGER.error("Error sending emails: " + ex); } submitInterfaceValidation(e, getAuthenticatedUser().getEmail(), repositoryInterface, false); return repositoryInterface; } @Override public RepositoryInterface updateRepositoryInterface(String repoId, String comment, RepositoryInterface repositoryInterface) throws Exception { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseurl()); this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompatibility()); this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet()); Repository repository = this.getRepositoryById(repoId); try { try { emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, authentication); emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, authentication); } catch (Exception e) { LOGGER.error("Error sending emails: " + e); } } catch (Exception e) { LOGGER.warn("Could not send emails", e); } submitInterfaceValidation(getRepositoryById(repoId), getAuthenticatedUser().getEmail(), repositoryInterface, true); return repositoryInterface; } private User getAuthenticatedUser() { return User.from(((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo()); } private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting) throws ValidatorServiceException { JobForValidation job = new JobForValidation(); job.setActivationId(UUID.randomUUID().toString()); job.setAdminEmails(Collections.singletonList(this.adminEmail)); job.setBaseUrl(iFace.getBaseurl()); job.setDatasourceId(repo.getId()); job.setDesiredCompatibilityLevel(iFace.getCompatibilityOverride()); job.setInterfaceId(iFace.getId()); job.setOfficialName(repo.getOfficialname()); job.setRepoType(repo.getEoscDatasourceType()); job.setUserEmail(userEmail); String accessSet = iFace.getAccessSet(); job.setValidationSet((accessSet.isEmpty() ? "none" : accessSet)); job.setRecords(-1); job.setRegistration(!updateExisting); job.setUpdateExisting(updateExisting); this.validatorService.submitJobForValidation(job); } private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) { iFace.setDatasource(repo.getId()); iFace.setContentdescription("metadata"); iFace.setCompatibility("UNKNOWN"); if (datatype.equals("re3data")) iFace.setAccessFormat("oai_datacite"); else iFace.setAccessFormat("oai_dc"); // FIXME: this will probably not work if (repo.getEoscDatasourceType() != null && !repo.getEoscDatasourceType().isEmpty()) iFace.setTypology(repo.getEoscDatasourceType()); else if (datatype.equalsIgnoreCase("journal")) iFace.setTypology("pubsrepository::journal"); else if (datatype.equalsIgnoreCase("aggregator")) iFace.setTypology("aggregator::pubsrepository::unknown"); else if (datatype.equalsIgnoreCase("opendoar")) iFace.setTypology("pubsrepository::unknown"); else if (datatype.equalsIgnoreCase("re3data")) iFace.setTypology("datarepository::unknown"); iFace.setRemovable(true); iFace.setProtocol("oai"); iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']"); iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8)); if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) { LOGGER.debug("set is empty: " + iFace.getAccessSet()); // iFace.removeAccessSet(); iFace.setAccessSet("none"); } return iFace; } @Override public List getDnetCountries() { LOGGER.debug("Getting dnet-countries!"); return converter.readFile("countries.txt"); } @Override public List getTypologies() { return converter.readFile("typologies.txt"); } @Override public List getTimezones() { List timezones = converter.readFile("timezones.txt"); return converter.toTimezones(timezones); } @Override public List getUrlsOfUserRepos(String userEmail, String page, String size) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/api/baseurl/") .path("/{page}/{size}") .build().expand(page, size).encode(); RequestFilter requestFilter = new RequestFilter(); requestFilter.setRegisteredby(userEmail); return Arrays.asList(restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class)); } private Vocabulary getVocabulary(String vocName) { if (!vocabularyMap.containsKey(vocName)) { vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT)); } return vocabularyMap.get(vocName); } @Override public Map getCompatibilityClasses(String mode) { LOGGER.debug("Getting compatibility classes for mode: " + mode); Map retMap = new HashMap(); Map compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap(); boolean foundData = false; for (Map.Entry entry : compatibilityClasses.entrySet()) { if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL)) return compatibilityClasses; else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) { if (entry.getKey().matches("^openaire[1-9].0_data$")) { retMap.put(entry.getKey(), entry.getValue()); foundData = true; } } else if (mode.equalsIgnoreCase("cris")) { if (entry.getKey().contains("openaire-cris")) { retMap.put(entry.getKey(), entry.getValue()); foundData = true; } } else { if (entry.getKey().matches("^openaire[1-9].0$") || entry.getKey().equals("driver")) retMap.put(entry.getKey(), entry.getValue()); } } //TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED if ((mode.equalsIgnoreCase("repository") || mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) && !foundData) retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)"); return retMap; } @Override public Map getDatasourceClasses(String mode) { LOGGER.debug("Getting datasource classes for mode: " + mode); Map retMap = new HashMap(); // TODO: refactor (remove?) for (Map.Entry entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) { if (mode.equalsIgnoreCase("aggregator")) { if (entry.getKey().contains("aggregator")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("journal")) { if (entry.getKey().contains("journal")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("opendoar")) { if (entry.getKey().contains("pubsrepository")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("re3data")) { if (entry.getKey().contains("datarepository")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("dris")) { if (entry.getKey().contains("crissystem")) retMap.put(entry.getKey(), entry.getValue()); } if (mode.equalsIgnoreCase("fairsharing")) { retMap.put(entry.getKey(), entry.getValue()); } } if (mode.equals("fairsharing")) { return retMap; } return filterResults(retMap, mode); } private Map filterResults(Map map, String mode) { HashMap filteredMap = new HashMap<>(); if (map != null && mode != null) { for (String key : map.keySet()) if (dataSourceClass.get(mode).contains(key)) filteredMap.put(key, map.get(key)); return filteredMap; } return Collections.emptyMap(); } @Override public String getCountryName(String countryCode) { return inverseCountriesMap.get(countryCode); } @Override public MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException { try { MetricsInfo metricsInfo = new MetricsInfo(); metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL); metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId))); return metricsInfo; } catch (Exception e) { LOGGER.error("Error while getting metrics info for repository: ", e); throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR); } } @Override public Map getListLatestUpdate(String mode) throws JSONException { Map dates = new HashMap<>(); if (mode.equals("repository")) { dates.put("opendoar", converter.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate())); dates.put("fairsharing", converter.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate())); // create re3data last collection date // dates.put("re3data", converter.toString(getRepositoryInterface("openaire____::re3data").get(1).getLastCollectionDate())); List re3interfaces = getRepositoryInterface("openaire____::re3data"); String re3Date = null; for (RepositoryInterface interf : re3interfaces) { if (interf.getLastCollectionDate() != null) { re3Date = converter.toString(interf.getLastCollectionDate()); break; } } dates.put("re3data", re3Date); return dates; } else if (mode.equals("cris")) return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate())); else if (mode.equals("opendoar")) // TODO: remove this and else clause return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate())); else /* * first api of re3data has null value on collection date * */ return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate())); } private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/oaiset") .queryParam("dsId", repositoryId) .queryParam("apiId", repositoryInterfaceId) .queryParam("oaiSet", validationSet) .build().encode(); restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, null, ResponseEntity.class); } private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/baseurl") .queryParam("dsId", repositoryId) .queryParam("apiId", repositoryInterfaceId) .queryParam("baseUrl", baseUrl) .build().encode(); restTemplate.postForObject(uriComponents.toUri(), null, String.class); } private void updateCompliance(String repositoryId, String repositoryInterfaceId, String compliance) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/compliance") .queryParam("dsId", repositoryId) .queryParam("apiId", repositoryInterfaceId) .queryParam("compliance", compliance) .build().encode(); restTemplate.postForObject(uriComponents.toUri(), null, String.class); } private MetricsNumbers getMetricsNumbers(String openAIREID) throws BrokerException { //build the uri params UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks"); //create new template engine RestTemplate template = new RestTemplate(); template.getMessageConverters().add(new MappingJackson2HttpMessageConverter()); ResponseEntity resp; //communicate with endpoint resp = template.exchange( builder.build().encode().toUri(), HttpMethod.GET, null, new ParameterizedTypeReference() { }); return resp.getBody(); } private String getOpenAIREId(String repoId) { if (repoId != null && repoId.contains("::")) { return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]); } return null; } private UriComponents getAggregationHistory(String repoId) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/aggregationhistory/") .path(repoId) .build().expand(repoId).encode(); } private UriComponents searchDatasource(String page, String size) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchdetails/") .path("/{page}/{size}/") .queryParam("requestSortBy", "officialname") .queryParam("order", "ASCENDING") .build().expand(page, size).encode(); } private UriComponents searchSnipperDatasource(String page, String size) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchsnippet/") .path("/{page}/{size}/") .queryParam("requestSortBy", "officialname") .queryParam("order", "ASCENDING") .build().expand(page, size).encode(); } private UriComponents searchRegisteredDatasource(String requestSortBy, String order, String page, String size) { return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchregistered/") .path("/{page}/{size}/") .queryParam("requestSortBy", requestSortBy) .queryParam("order", order) .build().expand(page, size).encode(); } private String getRepositoryType(String typology) { return invertedDataSourceClass.get(typology); } private List getRoleIdsFromUserRoles(String userEmail) { Integer coPersonId = registryCalls.getCoPersonIdByEmail(userEmail); JsonArray roles; ArrayList roleIds = new ArrayList<>(); ArrayList couIds = new ArrayList<>(); if (coPersonId != null) { roles = registryCalls.getRolesWithStatus(coPersonId, AaiRegistryService.RoleStatus.ACTIVE); for (JsonElement role : roles) { JsonObject object = role.getAsJsonObject(); if (object.get("CouId") == null) { continue; } couIds.add(object.get("CouId").getAsInt()); } roleIds.addAll(registryCalls.getCouNames(couIds).values()); } return roleIds; } }