package eu.dnetlib.repo.manager.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import eu.dnetlib.api.functionality.ValidatorServiceException; import eu.dnetlib.domain.data.Repository; import eu.dnetlib.domain.data.RepositoryInterface; import eu.dnetlib.domain.enabling.Vocabulary; import eu.dnetlib.domain.functionality.validator.JobForValidation; import eu.dnetlib.repo.manager.domain.RepositorySnippet; import eu.dnetlib.repo.manager.domain.RequestFilter; import eu.dnetlib.repo.manager.exception.ResourceNotFoundException; import eu.dnetlib.repo.manager.shared.*; import eu.dnetlib.repo.manager.utils.Converter; import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader; import gr.uoa.di.driver.xml.repository.INTERFACE; import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.*; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import javax.annotation.PostConstruct; import java.io.IOException; import java.sql.Timestamp; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @Service("repositoryService") public class RepositoryServiceImpl implements RepositoryService { @Value("${api.baseAddress}") private String baseAddress; @Value("${services.repo-manager.adminEmail}") private String adminEmail; @Autowired RestTemplate restTemplate; private HttpHeaders httpHeaders; private final String[] vocabularyNames = {"dnet:countries", "dnet:datasource_typologies", "dnet:compatibilityLevel"}; private static final Logger LOGGER = Logger.getLogger(RepositoryServiceImpl.class); @Value("${services.repomanager.usageStatisticsDiagramsBaseURL}") private String usageStatisticsDiagramsBaseURL; @Value("${services.repomanager.usageStatisticsNumbersBaseURL}") private String usageStatisticsNumbersBaseURL; @Autowired private VocabularyLoader vocabularyLoader; @Autowired private PiWikService piWikService; @Autowired private EmailUtils emailUtils; @Autowired ValidatorService validatorService; private Map vocabularyMap = new ConcurrentHashMap(); private Map countriesMap = new HashMap<>(); private Map inverseCountriesMap = new HashMap<>(); private static Map> dataSourceClass = new HashMap>(){{ put("opendoar",new ArrayList(){{ add("pubsrepository::institutional"); add("pubsrepository::thematic"); add("pubsrepository::unknown"); add("pubsrepository::mock"); }}); put("re3data",new ArrayList(){{ add("datarepository::unknown"); }}); put("journal",new ArrayList(){{ add("pubsrepository::journal"); }}); put("aggregator",new ArrayList(){{ add("aggregator::pubsrepository::institutional"); add("aggregator::pubsrepository::journals"); add("aggregator::datarepository"); add("aggregator::pubsrepository::unknown"); }}); }}; private static Map invertedDataSourceClass = new HashMap(){{ put("pubsrepository::institutional","opendoar"); put("pubsrepository::thematic","opendoar"); put("pubsrepository::unknown","opendoar"); put("pubsrepository::mock","opendoar"); put("datarepository::unknown","re3data"); put("pubsrepository::journal","journal"); put("aggregator::pubsrepository::institutional","aggregator"); put("aggregator::pubsrepository::journals","aggregator"); put("aggregator::datarepository","aggregator"); put("aggregator::pubsrepository::unknown","aggregator"); }}; @PostConstruct private void init() { LOGGER.debug("Initialization method of repository api!"); LOGGER.debug("Updated version!"); httpHeaders = new HttpHeaders(); httpHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8); // httpHeaders.set("Content-Type", "application/json"); for (String vocName : vocabularyNames) { vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT)); } Country[] countries = getCountries(); for (Country c : countries) { countriesMap.put(c.getName(), c.getCode()); inverseCountriesMap.put(c.getCode(), c.getName()); } } @Override public Country[] getCountries() { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/countries") .build().encode(); return restTemplate.getForObject(uriComponents.toUri(), Country[].class); } @Override public List getRepositoriesByCountry(@PathVariable("country") String country, @PathVariable("mode") String mode, @RequestParam(value = "managed",required=false) Boolean managed) throws JSONException, IOException { LOGGER.debug("Getting repositories by country!"); int page = 0; int size = 100; List resultSet = new ArrayList<>(); ObjectMapper mapper = new ObjectMapper(); String filterKey = "UNKNOWN"; if (mode.equalsIgnoreCase("opendoar")) filterKey = "openaire____::opendoar"; else if (mode.equalsIgnoreCase("re3data")) filterKey = "openaire____::re3data"; LOGGER.debug("Country code equals : " + country); LOGGER.debug("Filter mode equals : " + filterKey); UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page),String.valueOf(size)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); requestFilter.setCollectedfrom(filterKey); try{ String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); while (jsonArray.length() > 0 ) { resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class))); page += 1; uriComponents = searchSnipperDatasource(String.valueOf(page),String.valueOf(size)); rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); } return resultSet; }catch (Exception e){ LOGGER.debug("Exception on getRepositoriesByCountry" , e); // emailUtils.reportException(e); throw e; } } public List searchRegisteredRepositories(String country, String typology, String englishName, String officialName, String requestSortBy, String order, int page, int pageSize) throws Exception { LOGGER.debug("Searching registered repositories"); List resultSet = new ArrayList<>(); ObjectMapper mapper = new ObjectMapper(); UriComponents uriComponents = searchRegisteredDatasource(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); requestFilter.setTypology(typology); requestFilter.setOfficialname(officialName); requestFilter.setEnglishname(englishName); try { String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class))); return resultSet; }catch (Exception e){ LOGGER.error("Error searching registered datasources" , e); throw e; } } private Repository updateRepositoryInfo(Repository r) throws JSONException { /* * from datasource class * we get the datasource type form the inverted map * */ r.setDatasourceType(getRepositoryType(r.getDatasourceClass())); r.setInterfaces(this.getRepositoryInterface(r.getId())); r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId())); r.setCountryName(getCountryName(r.getCountryCode())); return r; } private Collection getRepositoriesByMode(String mode, List rs) { List reps = new ArrayList<>(); for (Repository r : rs) { if (r.getCollectedFrom() != null && r.getCollectedFrom().equals(mode)) reps.add(r); } return reps; } @Override public List getRepositoriesOfUser(@PathVariable("userEmail") String userEmail, @PathVariable("page") String page, @PathVariable("size") String size) throws JSONException { LOGGER.debug("Retreiving repositories of user : " + userEmail ); UriComponents uriComponents = searchDatasource(page,size); RequestFilter requestFilter = new RequestFilter(); requestFilter.setRegisteredby(userEmail); try{ String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); List repos = Converter.jsonToRepositoryList(new JSONObject(rs)); for (Repository r : repos) this.updateRepositoryInfo(r); return repos; }catch (Exception e){ LOGGER.debug("Exception on getRepositoriesOfUser" , e); emailUtils.reportException(e); throw e; } } @Override public Repository getRepositoryById(@PathVariable("id") String id) throws JSONException,ResourceNotFoundException { LOGGER.debug("Retreiving repositories with id : " + id ); Repository repo = null; UriComponents uriComponents = searchDatasource("0","100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); try{ String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); if(jsonArray.length() == 0) throw new ResourceNotFoundException(); repo = Converter.jsonToRepositoryObject(jsonArray.getJSONObject(0)); return updateRepositoryInfo(repo); }catch (JSONException e){ LOGGER.debug("Exception on getRepositoryById" , e); emailUtils.reportException(e); throw e; } } @Override public List getRepositoryAggregations(@PathVariable("id") String id) throws JSONException { LOGGER.debug("Retreiving aggregations for repository with id : " + id ); UriComponents uriComponents = searchDatasource("0","100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); List aggregationHistory = new ArrayList<>(); try { String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); JSONObject repository = new JSONObject(rs); if(repository.getJSONArray("datasourceInfo").length() == 0) return aggregationHistory; aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0))); return aggregationHistory.size() == 0? aggregationHistory : aggregationHistory.stream() .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) .limit(20) .collect(Collectors.toList()); } catch (JSONException e) { LOGGER.debug("Exception on getRepositoryAggregations" , e); emailUtils.reportException(e); throw e; } } @Override public Map> getRepositoryAggregationsByYear(@PathVariable("id") String id) throws JSONException { LOGGER.debug("Retreiving aggregations (by year) for repository with id : " + id ); UriComponents uriComponents = searchDatasource("0","100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); List aggregationHistory = new ArrayList<>(); Map> aggregationByYear = new HashMap<>(); try { String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); JSONObject repository = new JSONObject(rs); if(repository.getJSONArray("datasourceInfo").length() == 0) return aggregationByYear; aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0))); return aggregationHistory.size() == 0? aggregationByYear:createYearMap(aggregationHistory); } catch (JSONException e) { LOGGER.debug("Exception on getRepositoryAggregations" , e); emailUtils.reportException(e); throw e; } } private Map> createYearMap(List aggregationHistory) { Map> aggregationByYear; aggregationHistory = aggregationHistory.stream() .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) .collect(Collectors.toList()); return aggregationHistory.stream() .collect(Collectors.groupingBy(AggregationDetails::getYear)); } @Override public List getRepositoriesByName(@PathVariable("name") String name, @PathVariable("page") String page, @PathVariable("size") String size) throws JSONException { LOGGER.debug("Retreiving repositories with official name : " + name ); UriComponents uriComponents = searchDatasource("0","100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setOfficialname(name); try{ String rs = restTemplate.postForObject(uriComponents.toUri(),requestFilter, String.class); List repos = Converter.jsonToRepositoryList(new JSONObject(rs)); for (Repository r : repos) updateRepositoryInfo(r); return repos; }catch (Exception e){ LOGGER.debug("Exception on getRepositoriesByName" , e); emailUtils.reportException(e); throw e; } } @Override public List getRepositoryInterface(@PathVariable("id") String id) throws JSONException { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/") .path("/{id}") .build().expand(id).encode(); try{ String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); return Converter.jsonToRepositoryInterfaceList(new JSONObject(rs)); }catch (Exception e ){ LOGGER.debug("Exception on getRepositoryInterface" , e); emailUtils.reportException(e); throw e; } } @Override public Repository addRepository(@RequestParam("datatype") String datatype, @RequestBody Repository repository) throws Exception { LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId()); repository.setCountryCode(countriesMap.get(repository.getCountryName())); repository.setActivationId(UUID.randomUUID().toString()); repository.setCollectedFrom("infrastruct_::openaire"); if (datatype.equals("journal")) { repository.setId("openaire____::issn" + repository.getIssn()); repository.setNamespacePrefix("issn" + repository.getIssn()); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); }else if (datatype.equals("aggregator")) { repository.setId("openaire____::" + com.unboundid.util.Base64.encode(repository.getOfficialName())); repository.setNamespacePrefix(DigestUtils.md5Hex(repository.getOfficialName()).substring(0,12)); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); }else { this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication()); } return repository; } /* update method acting as add -> send email with registration topic/body*/ private Repository latentUpdate(Repository repository, Authentication authentication) throws Exception { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/update/") .build() .encode(); try { String json_repository = Converter.repositoryObjectToJson(repository); LOGGER.debug("JSON to add(update) -> " + json_repository); // // // TODO delete these 3 lines // HttpHeaders temp = new HttpHeaders(); // temp.setContentType(MediaType.APPLICATION_JSON_UTF8); HttpEntity httpEntity = new HttpEntity(json_repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(),HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { emailUtils.sendUserRegistrationEmail(repository, authentication); emailUtils.sendAdminRegistrationEmail(repository, authentication); } else LOGGER.debug(responseEntity.getBody().toString()); return repository; } catch (Exception e) { LOGGER.debug("Exception on updateRepository" , e); emailUtils.reportException(e); throw e; } } @Override public Repository updateRepository(@RequestBody Repository repository,Authentication authentication) throws Exception { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/update/") .build() .encode(); try { String json_repository = Converter.repositoryObjectToJson(repository); LOGGER.debug("JSON to update -> " + json_repository); HttpEntity httpEntity = new HttpEntity(json_repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(),HttpMethod.POST, httpEntity , ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { emailUtils.sendUserUpdateRepositoryEmail(repository, authentication); emailUtils.sendAdminUpdateRepositoryEmail(repository, authentication); } else LOGGER.debug(responseEntity.getBody().toString()); return repository; } catch (Exception e) { LOGGER.debug("Exception on updateRepository" , e); emailUtils.reportException(e); throw e; } } private void updateInterface(String datatype,RepositoryInterface iFace) { //TODO call update base url //((DatasourceManagerService) this.dmService.getService()).updateBaseUrl(repo.getId(), iFace.getId(), iFace.getBaseUrl()); if (!iFace.getAccessSet().isEmpty()) { LOGGER.debug("set not empty: " + iFace.getAccessSet()); //TODO call update method for access params // ((DatasourceManagerService) this.dmService.getService()).updateAccessParam(repo.getId(), iFace.getId(), "set", iFace.getAccessSet(), false); } else { //TODO call deleteAccessParamOrExtraField //((DatasourceManagerService) this.dmService.getService()).deleteAccessParamOrExtraField(repo.getId(), iFace.getId(), "set"); } //TODO update content description //((DatasourceManagerService) this.dmService.getService()).updateContentDescription(repo.getId(), iFace.getId(), "metadata"); if (datatype.equals("re3data")) { //TODO call update access params // ((DatasourceManagerService) this.dmService.getService()).updateAccessParam(repo.getId(), iFace.getId(), "format", "oai_datacite", false); iFace.setAccessFormat("oai_datacite"); } else { //TODO call update access params //((DatasourceManagerService) this.dmService.getService()).updateAccessParam(repo.getId(), iFace.getId(), "format", "oai_dc", false); iFace.setAccessFormat("oai_dc"); } } private void storeRepository(Repository repository, Authentication authentication) throws Exception { Date utilDate = new Date(); Timestamp date = new Timestamp(utilDate.getTime()); repository.setDateOfCollection(date); repository.setAggregator("OPENAIRE"); repository.setCountryCode(countriesMap.get(repository.getCountryName())); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/add/") .build() .encode(); String json_repository = Converter.repositoryObjectToJson(repository); HttpEntity httpEntity = new HttpEntity (json_repository,httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(),HttpMethod.POST, httpEntity, ResponseEntity.class); if(responseEntity.getStatusCode().equals(HttpStatus.OK)) { emailUtils.sendUserRegistrationEmail(repository, authentication); emailUtils.sendAdminRegistrationEmail(repository, authentication); } else { LOGGER.debug(responseEntity.getBody().toString()); } } @Override public void deleteRepositoryInterface(@RequestParam("id") String id , @RequestParam("registeredBy") String registeredBy){ UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/") .path("/{id}") .build().expand(id).encode(); LOGGER.debug(uriComponents.toUri()); restTemplate.delete(uriComponents.toUri()); } @Override public RepositoryInterface addRepositoryInterface(@RequestParam("datatype") String datatype, @RequestParam("repoId") String repoId, @RequestParam("registeredBy") String registeredBy, @RequestBody RepositoryInterface repositoryInterface) throws JSONException,ResourceNotFoundException { try { Repository e = this.getRepositoryById(repoId); repositoryInterface = createRepositoryInterface(e,repositoryInterface,datatype); String json_interface = Converter.repositoryInterfaceObjectToJson(e,repositoryInterface); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/add/") .build() .encode(); HttpEntity httpEntity = new HttpEntity (json_interface,httpHeaders); restTemplate.postForObject(uriComponents.toUri(),httpEntity,String.class); return repositoryInterface; } catch (JSONException e) { LOGGER.debug("Exception on addRepositoryInterface" , e); emailUtils.reportException(e); throw e; } } @Override public RepositoryInterface updateRepositoryInterface(@RequestParam("repoId") String repoId, @RequestParam("registeredBy") String registeredBy, @RequestBody RepositoryInterface repositoryInterface) throws Exception { this.updateBaseUrl(repoId,repositoryInterface.getId(),repositoryInterface.getBaseUrl()); this.updateCompliance(repoId,repositoryInterface.getId(),repositoryInterface.getCompliance()); this.updateValidationSet(repoId,repositoryInterface.getId(),repositoryInterface.getAccessSet()); return repositoryInterface; } private void submitInterfaceValidation(Repository repo, String repoType, String userEmail, RepositoryInterface iFace) throws ValidatorServiceException { JobForValidation job = new JobForValidation(); job.setActivationId(UUID.randomUUID().toString()); job.setAdminEmails(Collections.singletonList(this.adminEmail)); job.setBaseUrl(iFace.getBaseUrl()); job.setDatasourceId(repo.getId()); job.setDesiredCompatibilityLevel(iFace.getDesiredCompatibilityLevel()); job.setInterfaceId(iFace.getId()); // job.setInterfaceIdOld(null); job.setOfficialName(repo.getOfficialName()); job.setRepoType(repoType); job.setUserEmail(userEmail); job.setValidationSet(iFace.getAccessSet()); job.setRecords(-1); job.setRegistration(true); job.setUpdateExisting(false); this.validatorService.submitJobForValidation(job); } private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) { iFace.setContentDescription("metadata"); iFace.setCompliance("UNKNOWN"); if (datatype.equals("re3data")) iFace.setAccessFormat("oai_datacite"); else iFace.setAccessFormat("oai_dc"); if (repo.getDatasourceClass() != null && !repo.getDatasourceClass().isEmpty()) iFace.setTypology(repo.getDatasourceClass()); else if (datatype.equalsIgnoreCase("journal")) iFace.setTypology("pubsrepository::journal"); else if (datatype.equalsIgnoreCase("aggregator")) iFace.setTypology("aggregator::pubsrepository::unknown"); else if (datatype.equalsIgnoreCase("opendoar")) iFace.setTypology("pubsrepository::unknown"); else if (datatype.equalsIgnoreCase("re3data")) iFace.setTypology("datarepository::unknown"); iFace.setRemovable(true); iFace.setAccessProtocol("oai"); iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']"); iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8)); if (iFace.getAccessSet().isEmpty()) { LOGGER.debug("set is empty: " + iFace.getAccessSet()); iFace.removeAccessSet(); } return iFace; } @Override public List getDnetCountries() { LOGGER.debug("Getting dnet-countries!"); return Converter.readFile("countries.txt"); } @Override public List getTypologies() { return Converter.readFile("typologies.txt"); } @Override public List getTimezones() { List timezones = Converter.readFile("timezones.txt"); return Converter.toTimezones(timezones); } @Override public List getUrlsOfUserRepos(@PathVariable("user_email") String userEmail, @PathVariable("page") String page, @PathVariable("size") String size) throws JSONException { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/api/baseurl/") .path("/{page}/{size}") .build().expand(page,size).encode(); try{ RequestFilter requestFilter = new RequestFilter(); requestFilter.setRegisteredby(userEmail); return Arrays.asList(restTemplate.postForObject(uriComponents.toUri(),requestFilter, String[].class)); }catch (Exception e){ LOGGER.debug("Exception on addRepositoryInterface" , e); emailUtils.reportException(e); throw e; } } @Override public List getDatasourceVocabularies(@PathVariable("mode") String mode) { List resultSet = new ArrayList<>(); for (Map.Entry entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) { if (mode.equalsIgnoreCase("aggregator")) { if (entry.getKey().contains("aggregator")) resultSet.add(entry.getValue()); } else if (mode.equalsIgnoreCase("journal")) { if (entry.getKey().contains("journal")) resultSet.add(entry.getValue()); } else if (mode.equalsIgnoreCase("opendoar")) { if (entry.getKey().contains("pubsrepository")) resultSet.add(entry.getValue()); } else if (mode.equalsIgnoreCase("re3data")) { if (entry.getKey().contains("datarepository")) resultSet.add(entry.getValue()); } } return resultSet; } private Vocabulary getVocabulary(String vocName) { if (!vocabularyMap.containsKey(vocName)) { vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT)); } return vocabularyMap.get(vocName); } @Override public Map getCompatibilityClasses(@PathVariable("mode") String mode) { LOGGER.debug("Getting compatibility classes for mode: " + mode); Map retMap = new HashMap(); Map compatibilityClasses = this.getVocabulary("dnet:compatibilityLevel").getAsMap(); boolean foundData = false; for (Map.Entry entry : compatibilityClasses.entrySet()) { if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_ALL)) return compatibilityClasses; else if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA)) { if (entry.getKey().matches("^openaire[1-9].0_data$")) { retMap.put(entry.getKey(), entry.getValue()); foundData = true; } } else { if (entry.getKey().matches("^openaire[1-9].0$") || entry.getKey().equals("driver")) retMap.put(entry.getKey(), entry.getValue()); } } //TODO TO BE REMOVED WHEN VOCABULARIES ARE UPDATED if (mode.equalsIgnoreCase(Constants.REPOSITORY_MODE_RE3DATA) && !foundData) retMap.put("openaire2.0_data", "OpenAIRE Data (funded, referenced datasets)"); return retMap; } @Override public Map getDatasourceClasses(@PathVariable("mode") String mode) { LOGGER.debug("Getting datasource classes for mode: " + mode); Map retMap = new HashMap(); for (Map.Entry entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) { if (mode.equalsIgnoreCase("aggregator")) { if (entry.getKey().contains("aggregator")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("journal")) { if (entry.getKey().contains("journal")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("opendoar")) { if (entry.getKey().contains("pubsrepository")) retMap.put(entry.getKey(), entry.getValue()); } else if (mode.equalsIgnoreCase("re3data")) { if (entry.getKey().contains("datarepository")) retMap.put(entry.getKey(), entry.getValue()); } } return filterResults(retMap,mode); } private Map filterResults(Map map,String mode) { HashMap filteredMap = new HashMap<>(); for(String key:map.keySet()) if(dataSourceClass.get(mode).contains(key)) filteredMap.put(key,map.get(key)); return filteredMap; } @Override public String getCountryName(String countryCode) { return inverseCountriesMap.get(countryCode); } @Override public MetricsInfo getMetricsInfoForRepository(@PathVariable("repoId") String repoId) throws RepositoryServiceException { try { MetricsInfo metricsInfo = new MetricsInfo(); metricsInfo.setDiagramsBaseURL(this.usageStatisticsDiagramsBaseURL); metricsInfo.setMetricsNumbers(getMetricsNumbers(getOpenAIREId(repoId))); return metricsInfo; } catch (Exception e) { LOGGER.error("Error while getting metrics info for repository: ", e); emailUtils.reportException(e); throw new RepositoryServiceException("General error", RepositoryServiceException.ErrorCode.GENERAL_ERROR); } } @Override public Map getListLatestUpdate(@PathVariable("mode") String mode) throws JSONException { if(mode.equals("opendoar")) return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::"+mode).get(0).getLastCollectionDate()); else /* * first api of re3data has null value on collection date * */ return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::"+mode).get(1).getLastCollectionDate()); } private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/oaiset") .queryParam("dsId",repositoryId) .queryParam("apiId",repositoryInterfaceId) .queryParam("oaiSet",validationSet) .build().encode(); restTemplate.exchange(uriComponents.toUri(),HttpMethod.POST, null, ResponseEntity.class); } private void updateBaseUrl(String repositoryId, String repositoryInterfaceId, String baseUrl) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/baseurl") .queryParam("dsId",repositoryId) .queryParam("apiId",repositoryInterfaceId) .queryParam("baseUrl",baseUrl) .build().encode(); restTemplate.postForObject(uriComponents.toUri(),null,String.class); } private void updateCompliance(String repositoryId, String repositoryInterfaceId,String compliance) { UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/compliance") .queryParam("dsId",repositoryId) .queryParam("apiId",repositoryInterfaceId) .queryParam("compliance",compliance) .build().encode(); restTemplate.postForObject(uriComponents.toUri(),null,String.class); } private MetricsNumbers getMetricsNumbers(String openAIREID) throws BrokerException { //build the uri params UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(this.usageStatisticsNumbersBaseURL + openAIREID + "/clicks"); //create new template engine RestTemplate template = new RestTemplate(); template.getMessageConverters().add(new MappingJackson2HttpMessageConverter()); ResponseEntity resp; try { //communicate with endpoint resp = template.exchange( builder.build().encode().toUri(), HttpMethod.GET, null, new ParameterizedTypeReference() { }); } catch (RestClientException e) { LOGGER.debug("Exception on getMetricsNumbers" , e); emailUtils.reportException(e); throw e; } return resp.getBody(); } private String getOpenAIREId(String repoId) { if (repoId != null && repoId.contains("::")) { return repoId.split("::")[0] + "::" + DigestUtils.md5Hex(repoId.split("::")[1]); } return null; } private UriComponents searchDatasource(String page,String size){ return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/search/") .path("/{page}/{size}/") .queryParam("requestSortBy","officialname") .queryParam("order","ASCENDING") .build().expand(page, size).encode(); } private UriComponents searchSnipperDatasource(String page,String size){ return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchsnippet/") .path("/{page}/{size}/") .queryParam("requestSortBy","officialname") .queryParam("order","ASCENDING") .build().expand(page, size).encode(); } private UriComponents searchRegisteredDatasource(String requestSortBy, String order, String page,String size){ return UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/searchregistered/") .path("/{page}/{size}/") .queryParam("requestSortBy",requestSortBy) .queryParam("order",order) .build().expand(page, size).encode(); } private String getRepositoryType(String typology){ return invertedDataSourceClass.get(typology); } }