diff --git a/pom.xml b/pom.xml index 1225f8f..e30cd3e 100644 --- a/pom.xml +++ b/pom.xml @@ -137,21 +137,21 @@ - - eu.dnetlib - uoa-domain - [2.0.0-SNAPSHOT, 3.0.0) - - - cglib - cglib - - - log4j - log4j - - - + + + + + + + + + + + + + + + eu.dnetlib dnet-openaire-usage-stats-sushilite @@ -291,7 +291,7 @@ org.springframework.session spring-session-data-redis - + - - -org.apache.commons -commons-pool2 - + ${jedis.version} + + + + org.apache.commons + commons-pool2 + com.google.code.gson gson @@ -327,8 +327,8 @@ com.netflix.hystrix hystrix-core 1.5.18 - - dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal")); } else if (key.contains("pubsrepository")) { // do not change order <-- @@ -175,9 +179,9 @@ public class RepositoryServiceImpl implements RepositoryService { for (String repoId : ids) { requestFilter.setId(repoId); - String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); + List rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, List.class); - repos.addAll(Converter.jsonToRepositoryList(new JSONObject(rs))); +// repos.addAll(converter.toRepositoryList(new JSONObject(rs))); } for (Repository r : repos) @@ -239,12 +243,10 @@ public class RepositoryServiceImpl implements RepositoryService { ObjectMapper mapper = new ObjectMapper(); String filterKey = "UNKNOWN"; - if (mode.equalsIgnoreCase("opendoar")) - filterKey = "openaire____::opendoar"; - else if (mode.equalsIgnoreCase("re3data")) - filterKey = "openaire____::re3data"; + if (mode.equalsIgnoreCase("repository")) + filterKey = "Repository"; else if (mode.equalsIgnoreCase("cris")) - filterKey = "eurocrisdris::dris"; + filterKey = "CRIS system"; LOGGER.debug("Country code equals : " + country); @@ -253,7 +255,7 @@ public class RepositoryServiceImpl implements RepositoryService { UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size)); RequestFilter requestFilter = new RequestFilter(); requestFilter.setCountry(country); - requestFilter.setCollectedfrom(filterKey); + requestFilter.setEoscDatasourceType(filterKey); String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); @@ -309,15 +311,8 @@ public class RepositoryServiceImpl implements RepositoryService { } private Repository updateRepositoryInfo(Repository r) throws JSONException { - - /* - * from datasource class - * we get the datasource type form the inverted map - * */ - r.setDatasourceType(getRepositoryType(r.getDatasourceClass())); r.setInterfaces(this.getRepositoryInterface(r.getId())); r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId())); - r.setCountryName(getCountryName(r.getCountryCode())); return r; } @@ -338,21 +333,33 @@ public class RepositoryServiceImpl implements RepositoryService { @Override public List getRepositoriesSnippetsOfUser(String page, String size) throws Exception { - Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles()); - return getRepositoriesSnippets(new ArrayList<>(repoIds)); + return getRepositoriesSnippetsOfUser(null, page, size); } @Override public List getRepositoriesSnippetsOfUser(String userEmail, String page, String size) throws Exception { - Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail)); - return getRepositoriesSnippets(new ArrayList<>(repoIds)); + int from = Integer.parseInt(page) * Integer.parseInt(size); + int to = from + Integer.parseInt(size); + List repoIds = new ArrayList<>(); + if (userEmail != null && !"".equals(userEmail)) { + repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail))); + } else { + repoIds.addAll(roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles())); + } + + if (repoIds.size() < from) { + return Collections.emptyList(); + } else if (repoIds.size() < to) { + to = repoIds.size(); + } + return getRepositoriesSnippets(repoIds.subList(from, to)); // FIXME: returns less results if some repos are not found } @Override public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException { LOGGER.debug("Retrieving repositories with id : " + id); - RepositorySnippet repo = null; + RepositorySnippet repo; UriComponents uriComponents = searchSnipperDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); @@ -363,7 +370,7 @@ public class RepositoryServiceImpl implements RepositoryService { if (jsonArray.length() == 0) throw new ResourceNotFoundException(); - repo = Converter.jsonToRepositorySnippetObject(jsonArray.getJSONObject(0)); + repo = converter.toRepositorySnippet(jsonArray.getJSONObject(0)); return repo; } @@ -371,19 +378,25 @@ public class RepositoryServiceImpl implements RepositoryService { public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException { LOGGER.debug("Retrieving repositories with id : " + id); - Repository repo = null; + Repository repo; UriComponents uriComponents = searchDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setId(id); - String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); - JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); +// String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); +// JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo"); - if (jsonArray.length() == 0) + DatasourceResponse response; + response = (DatasourceResponse) restTemplate.postForObject(uriComponents.toUri(), requestFilter, DatasourceResponse.class); + List datasources = response.getDatasourceInfo(); + if (datasources.size() == 0) throw new ResourceNotFoundException(); - repo = Converter.jsonToRepositoryObject(jsonArray.getJSONObject(0)); - return updateRepositoryInfo(repo); +// repo = converter.toRepository(jsonArray.getJSONObject(0)); +// return updateRepositoryInfo(repo); + + return updateRepositoryInfo(converter.toRepository(datasources.get(0))); + } @@ -396,12 +409,9 @@ public class RepositoryServiceImpl implements RepositoryService { String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); JSONArray aggregationInfo = new JSONObject(rs).getJSONArray("aggregationInfo"); - List aggregationHistory = new ArrayList<>(Converter.getAggregationHistoryFromJson(aggregationInfo)); + List aggregationHistory = new ArrayList<>(converter.toAggregationHistory(aggregationInfo)); return aggregationHistory; -// return aggregationHistory.size() == 0 ? aggregationHistory : aggregationHistory.stream() -// .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) -// .collect(Collectors.toList()); } @Override @@ -444,7 +454,7 @@ public class RepositoryServiceImpl implements RepositoryService { requestFilter.setOfficialname(name); String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); - List repos = Converter.jsonToRepositoryList(new JSONObject(rs)); + List repos = converter.toRepositoryList(new JSONObject(rs)); for (Repository r : repos) updateRepositoryInfo(r); return repos; @@ -459,8 +469,10 @@ public class RepositoryServiceImpl implements RepositoryService { .path("/{id}") .build().expand(id).encode(); - String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); - return Converter.jsonToRepositoryInterfaceList(new JSONObject(rs)); +// String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); + ApiDetailsResponse rs = restTemplate.getForObject(uriComponents.toUri(), ApiDetailsResponse.class); + + return converter.toRepositoryInterfaceList(rs.getApi()); } @@ -469,26 +481,37 @@ public class RepositoryServiceImpl implements RepositoryService { LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId()); - repository.setCountryCode(countriesMap.get(repository.getCountryName())); repository.setActivationId(UUID.randomUUID().toString()); - repository.setCollectedFrom("infrastruct_::openaire"); + repository.setCollectedfrom("infrastruct_::openaire"); + +// Date now = new Date(); +// repository.setRegistrationdate(now); +// repository.setConsentTermsOfUseDate(now); +// repository.setLastConsentTermsOfUseDate(now); if (datatype.equals("journal")) { + repository.setEoscDatasourceType("Journal archive"); repository.setId("openaire____::issn" + repository.getIssn()); - repository.setNamespacePrefix("issn" + repository.getIssn()); + repository.setNamespaceprefix("issn" + repository.getIssn()); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); } else if (datatype.equals("aggregator")) { - repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialName())); - repository.setNamespacePrefix(DigestUtils.md5Hex(repository.getOfficialName()).substring(0, 12)); + repository.setEoscDatasourceType("Aggregator"); + repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialname())); + repository.setNamespaceprefix(DigestUtils.md5Hex(repository.getOfficialname()).substring(0, 12)); this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication()); } else { + if (repository.getTypology().contains("crissystem")) { + repository.setEoscDatasourceType("CRIS system"); + } else { + repository.setEoscDatasourceType("Repository"); + } this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication()); } // TODO: move the following code elsewhere (creation and assignment of role to user) ?? // Create new role String newRoleName = roleMappingService.getRoleIdByRepoId(repository.getId()); - Role newRole = new Role(newRoleName, repository.getOfficialName()); + Role newRole = new Role(newRoleName, repository.getOfficialname()); Integer couId = null; try { couId = registryCalls.createRole(newRole); @@ -529,10 +552,11 @@ public class RepositoryServiceImpl implements RepositoryService { .build() .encode(); - String json_repository = Converter.repositoryObjectToJson(repository); - LOGGER.debug("JSON to add(update) -> " + json_repository); + // FIXME: problematic +// String json_repository = converter.toJson(repository); +// LOGGER.debug("JSON to add(update) -> " + json_repository); - HttpEntity httpEntity = new HttpEntity<>(json_repository, httpHeaders); + HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); // TODO: check if it works (Repository contains extra fields) ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { @@ -555,11 +579,11 @@ public class RepositoryServiceImpl implements RepositoryService { .build() .encode(); - String json_repository = Converter.repositoryObjectToJson(repository); + // FIXME: problematic +// String json_repository = converter.toJson(repository); +// LOGGER.debug("JSON to update -> " + json_repository); - LOGGER.debug("JSON to update -> " + json_repository); - - HttpEntity httpEntity = new HttpEntity(json_repository, httpHeaders); + HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity , ResponseEntity.class); @@ -580,16 +604,16 @@ public class RepositoryServiceImpl implements RepositoryService { Date utilDate = new Date(); Timestamp date = new Timestamp(utilDate.getTime()); - repository.setDateOfCollection(date); + repository.setDateofcollection(date); repository.setAggregator("OPENAIRE"); - repository.setCountryCode(countriesMap.get(repository.getCountryName())); +// repository.setCountryCode(countriesMap.get(repository.getCountryName())); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/add/") .build() .encode(); - String json_repository = Converter.repositoryObjectToJson(repository); - HttpEntity httpEntity = new HttpEntity(json_repository, httpHeaders); +// String json_repository = converter.toJson(repository); + HttpEntity httpEntity = new HttpEntity<>(repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { @@ -622,14 +646,14 @@ public class RepositoryServiceImpl implements RepositoryService { String comment, RepositoryInterface repositoryInterface) throws Exception { Repository e = this.getRepositoryById(repoId); repositoryInterface = createRepositoryInterface(e, repositoryInterface, datatype); - String json_interface = Converter.repositoryInterfaceObjectToJson(e, repositoryInterface); +// String json_interface = converter.toJson(e, repositoryInterface); UriComponents uriComponents = UriComponentsBuilder .fromHttpUrl(baseAddress + "/ds/api/add/") .build() .encode(); - HttpEntity httpEntity = new HttpEntity<>(json_interface, httpHeaders); + HttpEntity httpEntity = new HttpEntity<>(repositoryInterface, httpHeaders); restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class); @@ -650,8 +674,8 @@ public class RepositoryServiceImpl implements RepositoryService { String registeredBy, String comment, RepositoryInterface repositoryInterface) throws Exception { - this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseUrl()); - this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompliance()); + this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseurl()); + this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompatibility()); this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet()); Repository repository = this.getRepositoryById(repoId); @@ -676,12 +700,12 @@ public class RepositoryServiceImpl implements RepositoryService { job.setActivationId(UUID.randomUUID().toString()); job.setAdminEmails(Collections.singletonList(this.adminEmail)); - job.setBaseUrl(iFace.getBaseUrl()); + job.setBaseUrl(iFace.getBaseurl()); job.setDatasourceId(repo.getId()); - job.setDesiredCompatibilityLevel(iFace.getDesiredCompatibilityLevel()); + job.setDesiredCompatibilityLevel(iFace.getCompatibilityOverride()); job.setInterfaceId(iFace.getId()); - job.setOfficialName(repo.getOfficialName()); - job.setRepoType(repo.getDatasourceType()); + job.setOfficialName(repo.getOfficialname()); + job.setRepoType(repo.getEoscDatasourceType()); job.setUserEmail(userEmail); job.setValidationSet((iFace.getAccessSet().isEmpty() ? "none" : iFace.getAccessSet())); job.setRecords(-1); @@ -693,8 +717,9 @@ public class RepositoryServiceImpl implements RepositoryService { private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) { - iFace.setContentDescription("metadata"); - iFace.setCompliance("UNKNOWN"); + iFace.setDatasource(repo.getId()); + iFace.setContentdescription("metadata"); + iFace.setCompatibility("UNKNOWN"); if (datatype.equals("re3data")) iFace.setAccessFormat("oai_datacite"); @@ -702,8 +727,9 @@ public class RepositoryServiceImpl implements RepositoryService { iFace.setAccessFormat("oai_dc"); - if (repo.getDatasourceClass() != null && !repo.getDatasourceClass().isEmpty()) - iFace.setTypology(repo.getDatasourceClass()); + // FIXME: this will probably not work + if (repo.getEoscDatasourceType() != null && !repo.getEoscDatasourceType().isEmpty()) + iFace.setTypology(repo.getEoscDatasourceType()); else if (datatype.equalsIgnoreCase("journal")) iFace.setTypology("pubsrepository::journal"); else if (datatype.equalsIgnoreCase("aggregator")) @@ -714,12 +740,12 @@ public class RepositoryServiceImpl implements RepositoryService { iFace.setTypology("datarepository::unknown"); iFace.setRemovable(true); - iFace.setAccessProtocol("oai"); + iFace.setProtocol("oai"); iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']"); iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8)); if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) { LOGGER.debug("set is empty: " + iFace.getAccessSet()); - iFace.removeAccessSet(); +// iFace.removeAccessSet(); iFace.setAccessSet("none"); } return iFace; @@ -728,18 +754,18 @@ public class RepositoryServiceImpl implements RepositoryService { @Override public List getDnetCountries() { LOGGER.debug("Getting dnet-countries!"); - return Converter.readFile("countries.txt"); + return converter.readFile("countries.txt"); } @Override public List getTypologies() { - return Converter.readFile("typologies.txt"); + return converter.readFile("typologies.txt"); } @Override public List getTimezones() { - List timezones = Converter.readFile("timezones.txt"); - return Converter.toTimezones(timezones); + List timezones = converter.readFile("timezones.txt"); + return converter.toTimezones(timezones); } @Override @@ -756,33 +782,6 @@ public class RepositoryServiceImpl implements RepositoryService { return Arrays.asList(restTemplate.postForObject(uriComponents.toUri(), requestFilter, String[].class)); } - @Override - public List getDatasourceVocabularies(String mode) { - - List resultSet = new ArrayList<>(); - for (Map.Entry entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) { - if (mode.equalsIgnoreCase("aggregator")) { - if (entry.getKey().contains("aggregator")) - resultSet.add(entry.getValue()); - } else if (mode.equalsIgnoreCase("journal")) { - if (entry.getKey().contains("journal")) - resultSet.add(entry.getValue()); - } else if (mode.equalsIgnoreCase("opendoar")) { - if (entry.getKey().contains("pubsrepository")) - resultSet.add(entry.getValue()); - } else if (mode.equalsIgnoreCase("re3data")) { - if (entry.getKey().contains("datarepository")) - resultSet.add(entry.getValue()); - } else if (mode.equalsIgnoreCase("cris")) { - if (entry.getKey().contains("crissystem")) - resultSet.add(entry.getValue()); - } - } - - - return resultSet; - } - private Vocabulary getVocabulary(String vocName) { if (!vocabularyMap.containsKey(vocName)) { @@ -834,6 +833,7 @@ public class RepositoryServiceImpl implements RepositoryService { Map retMap = new HashMap(); + // TODO: refactor (remove?) for (Map.Entry entry : this.getVocabulary("dnet:datasource_typologies").getAsMap().entrySet()) { if (mode.equalsIgnoreCase("aggregator")) { if (entry.getKey().contains("aggregator")) @@ -847,10 +847,16 @@ public class RepositoryServiceImpl implements RepositoryService { } else if (mode.equalsIgnoreCase("re3data")) { if (entry.getKey().contains("datarepository")) retMap.put(entry.getKey(), entry.getValue()); - } else if (mode.equalsIgnoreCase("cris")) { + } else if (mode.equalsIgnoreCase("dris")) { if (entry.getKey().contains("crissystem")) retMap.put(entry.getKey(), entry.getValue()); } + if (mode.equalsIgnoreCase("fairsharing")) { + retMap.put(entry.getKey(), entry.getValue()); + } + } + if (mode.equals("fairsharing")) { + return retMap; } return filterResults(retMap, mode); @@ -859,11 +865,14 @@ public class RepositoryServiceImpl implements RepositoryService { private Map filterResults(Map map, String mode) { HashMap filteredMap = new HashMap<>(); - for (String key : map.keySet()) - if (dataSourceClass.get(mode).contains(key)) - filteredMap.put(key, map.get(key)); + if (map != null && mode != null) { + for (String key : map.keySet()) + if (dataSourceClass.get(mode).contains(key)) + filteredMap.put(key, map.get(key)); - return filteredMap; + return filteredMap; + } + return Collections.emptyMap(); } @Override @@ -888,15 +897,22 @@ public class RepositoryServiceImpl implements RepositoryService { @Override public Map getListLatestUpdate(String mode) throws JSONException { - if (mode.equals("opendoar")) - return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()); + Map dates = new HashMap<>(); + if (mode.equals("repository")) { + dates.put("opendoar", converter.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate())); + dates.put("re3data", converter.toString(getRepositoryInterface("openaire____::re3data").get(1).getLastCollectionDate())); + dates.put("fairsharing", converter.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate())); + return dates; + } else if (mode.equals("cris")) - return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate()); + return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate())); + else if (mode.equals("opendoar")) // TODO: remove this and else clause + return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate())); else /* * first api of re3data has null value on collection date * */ - return Collections.singletonMap("lastCollectionDate", getRepositoryInterface("openaire____::" + mode).get(1).getLastCollectionDate()); + return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate())); } private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception { diff --git a/src/main/java/eu/dnetlib/repo/manager/service/ValidatorService.java b/src/main/java/eu/dnetlib/repo/manager/service/ValidatorService.java index 5b6c93f..5541e1e 100644 --- a/src/main/java/eu/dnetlib/repo/manager/service/ValidatorService.java +++ b/src/main/java/eu/dnetlib/repo/manager/service/ValidatorService.java @@ -5,7 +5,7 @@ import eu.dnetlib.domain.functionality.validator.JobForValidation; import eu.dnetlib.domain.functionality.validator.RuleSet; import eu.dnetlib.domain.functionality.validator.StoredJob; import eu.dnetlib.repo.manager.domain.InterfaceInformation; -import eu.dnetlib.repo.manager.domain.ValidationServiceException; +import eu.dnetlib.repo.manager.exception.ValidationServiceException; import eu.dnetlib.repo.manager.exception.ResourceNotFoundException; import org.json.JSONException; import org.springframework.http.ResponseEntity; diff --git a/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java b/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java index fdf4441..b9741e9 100644 --- a/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java +++ b/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java @@ -1,11 +1,11 @@ package eu.dnetlib.repo.manager.service; import eu.dnetlib.api.functionality.ValidatorServiceException; -import eu.dnetlib.domain.data.RepositoryInterface; +import eu.dnetlib.repo.manager.domain.RepositoryInterface; import eu.dnetlib.domain.functionality.validator.*; import eu.dnetlib.repo.manager.domain.Constants; import eu.dnetlib.repo.manager.domain.InterfaceInformation; -import eu.dnetlib.repo.manager.domain.ValidationServiceException; +import eu.dnetlib.repo.manager.exception.ValidationServiceException; import eu.dnetlib.repo.manager.utils.CrisValidatorUtils; import eu.dnetlib.repo.manager.utils.OaiTools; import gr.uoa.di.driver.util.ServiceLocator; @@ -293,7 +293,11 @@ public class ValidatorServiceImpl implements ValidatorService { @Override public List getJobsSummary(String repoId, int limit) throws JSONException, ValidatorServiceException { - return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit); + return getValidationService().getJobSummary( + repositoryService.getRepositoryInterface(repoId) + .stream() + .map(RepositoryInterface::getBaseurl) + .collect(Collectors.toList()), limit); } } diff --git a/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java b/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java index cac17ee..cca43fe 100644 --- a/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java +++ b/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java @@ -2,11 +2,7 @@ package eu.dnetlib.repo.manager.utils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.domain.data.Repository; -import eu.dnetlib.domain.data.RepositoryInterface; -import eu.dnetlib.repo.manager.domain.AggregationDetails; -import eu.dnetlib.repo.manager.domain.RepositorySnippet; -import eu.dnetlib.repo.manager.domain.Timezone; +import eu.dnetlib.repo.manager.domain.*; import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.json.JSONArray; @@ -27,367 +23,67 @@ public class Converter { private static final Logger LOGGER = Logger.getLogger(Converter.class); - public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException { + private final ObjectMapper objectMapper; - Repository repository = new Repository(); + public Converter() { + objectMapper = new ObjectMapper(); + } + public Repository toRepository(Object repositoryObject) { -// JSONObject datasource = repositoryObject.getJSONObject("datasource"); - JSONObject datasource = repositoryObject; - - //if( datasource.equals(null)) - // return null; - - repository.setId(datasource.get("id").toString()); - repository.setOfficialName(datasource.get("officialname").toString()); - - repository.setEnglishName(datasource.get("englishname").toString()); - if (repository.getEnglishName().equals("null")) - repository.setEnglishName(""); - - repository.setWebsiteUrl(datasource.get("websiteurl").toString()); - if (repository.getWebsiteUrl().equals("null")) - repository.setWebsiteUrl(""); - - repository.setLogoUrl(datasource.get("logourl").toString()); - if (repository.getLogoUrl().equals("null")) - repository.setLogoUrl(""); - - repository.setContactEmail(datasource.get("contactemail").toString()); - if (repository.getContactEmail().equals("null")) - repository.setContactEmail(""); - - - repository.setLatitude(toDouble(datasource.get("latitude").toString())); - repository.setLongitude(toDouble(datasource.get("longitude").toString())); - Double timezone = toDouble(datasource.get("timezone").toString()); - repository.setTimezone(timezone != null ? timezone : 0.0); - repository.setNamespacePrefix(datasource.get("namespaceprefix").toString()); - repository.setOdLanguages(datasource.get("languages").toString()); - repository.setDateOfValidation(convertStringToDate(datasource.get("dateofvalidation").toString())); - - /* typology -> platform - * datasource class -> typology */ - repository.setTypology(datasource.get("platform").toString()); - if (repository.getTypology().equals("null")) - repository.setTypology(""); - -// // TODO: enable in future release -// /* 07-04-2022 | "typology" -> "eoscDatasourceType" */ -// try { // FIXME: remove attemp to get typology if eoscDatasourceType fails -// repository.setDatasourceClass(datasource.get("eoscDatasourceType").toString()); -// } catch (JSONException e) { -// repository.setDatasourceClass(datasource.get("typology").toString()); -// } - repository.setDatasourceClass(datasource.get("typology").toString()); -// <-- - - repository.setDateOfCollection(convertStringToDate(datasource.get("dateofcollection").toString())); - repository.setActivationId(datasource.get("activationId").toString()); - - repository.setDescription(datasource.get("description").toString()); - if (repository.getDescription().equals("null")) - repository.setDescription(""); - - repository.setIssn(datasource.get("issn").toString()); - repository.setLissn(datasource.get("lissn").toString()); - if (repository.getLissn().equals("null")) - repository.setLissn(""); - repository.setEissn(datasource.get("eissn").toString()); - if (repository.getEissn().equals("null")) - repository.setEissn(""); - repository.setRegisteredBy(datasource.get("registeredby").toString()); - - /* managed field */ - repository.setRegistered(Boolean.parseBoolean(datasource.get("managed").toString())); - - //subjects - - repository.setAggregator(datasource.get("aggregator").toString()); - repository.setCollectedFrom(datasource.get("collectedfrom").toString()); - - //TODO change organization to list - JSONArray organizations = ((JSONArray) datasource.get("organizations")); - if (organizations.length() != 0) { - repository.setOrganization(((JSONArray) datasource.get("organizations")).getJSONObject(0).get("legalname").toString()); - String countryCode = ((JSONArray) datasource.get("organizations")).getJSONObject(0).get("country").toString(); - repository.setCountryCode(countryCode); - } - - repository.setConsentTermsOfUse(convertStringToBoolean(datasource.get("consentTermsOfUse").toString())); - repository.setConsentTermsOfUseDate(null); - repository.setLastConsentTermsOfUseDate(null); - try { - repository.setConsentTermsOfUseDate(convertStringToDate(datasource.get("consentTermsOfUseDate").toString())); - repository.setLastConsentTermsOfUseDate(convertStringToDate(datasource.get("lastConsentTermsOfUseDate").toString())); - } catch (JSONException e) { - LOGGER.info("Error setting consentTermsOfUseDate date and lastConsentTermsOfUseDate", e); - } - repository.setFullTextDownload(convertStringToBoolean(datasource.get("fullTextDownload").toString())); - - /* identities field */ + Repository repository = objectMapper.convertValue(repositoryObject, Repository.class); return repository; } - public static Boolean convertStringToBoolean(String value) { - return value.equals("null") ? null : Boolean.valueOf(value); + public RepositorySnippet toRepositorySnippet(JSONObject repositorySnippetObject) { + + RepositorySnippet snippet = objectMapper.convertValue(repositorySnippetObject, RepositorySnippet.class); + + return snippet; } - public static Date convertStringToDate(String date) { - - if (Objects.equals(date, "null")) - return null; - - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - try { - return formatter.parse(date); - } catch (ParseException e) { - LOGGER.error(e); - } - return null; - } - - public static String convertDateToString(Date date) { - - if (Objects.equals(date, null)) - return null; - - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - return formatter.format(date); - } - - public static Double toDouble(String number) { - if (Objects.equals(number, "null")) - return 0.0; - else - return Double.valueOf(number); - } - - public static List jsonToRepositorySnippetList(JSONObject json) throws JSONException { - - List resultSet = new ArrayList<>(); - JSONArray rs = json.getJSONArray("datasourceInfo"); - for (int i = 0; i < rs.length(); i++) - resultSet.add(jsonToRepositorySnippetObject(rs.getJSONObject(i))); - return resultSet; - } - - public static RepositorySnippet jsonToRepositorySnippetObject(JSONObject repositorySnippetObject) throws JSONException { - - - RepositorySnippet repositorySnippet = new RepositorySnippet(); - -// JSONObject datasource = repositorySnippetObject.getJSONObject("datasource"); - - - repositorySnippet.setId(repositorySnippetObject.get("id").toString()); - repositorySnippet.setOfficialname(repositorySnippetObject.get("officialname").toString()); - - repositorySnippet.setEnglishname(repositorySnippetObject.get("englishname").toString()); - if (repositorySnippet.getEnglishname().equals("null")) - repositorySnippet.setEnglishname(""); - - repositorySnippet.setWebsiteurl(repositorySnippetObject.get("websiteurl").toString()); - if (repositorySnippet.getWebsiteurl().equals("null")) - repositorySnippet.setWebsiteurl(""); - - repositorySnippet.setRegisteredby(repositorySnippetObject.get("registeredby").toString()); - if (repositorySnippet.getRegisteredby().equals("null")) - repositorySnippet.setRegisteredby(""); - - repositorySnippet.setConsentTermsOfUse(repositorySnippetObject.get("consenttermsofuse").toString()); - repositorySnippet.setFullTextDownload(repositorySnippetObject.get("fulltextdownload").toString()); - repositorySnippet.setConsentTermsOfUseDate(convertStringToDate(repositorySnippetObject.get("consenttermsofusedate").toString())); - - return repositorySnippet; - - } - - public static List jsonToRepositoryList(JSONObject json) throws JSONException { + public List toRepositoryList(JSONObject json) throws JSONException { List resultSet = new ArrayList<>(); JSONArray rs = json.getJSONArray("datasourceInfo"); for (int i = 0; i < rs.length(); i++) - resultSet.add(jsonToRepositoryObject(rs.getJSONObject(i))); + resultSet.add(toRepository(rs.getJSONObject(i))); return resultSet; } - public static List jsonToRepositoryInterfaceList(JSONObject json) throws JSONException { + public List toRepositoryInterfaceList(JSONObject json) throws JSONException { List resultSet = new ArrayList<>(); JSONArray rs = json.getJSONArray("api"); for (int i = 0; i < rs.length(); i++) - resultSet.add(jsonToRepositoryInterfaceObject(rs.getJSONObject(i))); + resultSet.add(toRepositoryInterface(rs.getJSONObject(i))); return resultSet; } - public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException { + public List toRepositoryInterfaceList(List apiDetailsList) throws JSONException { - RepositoryInterface repositoryInterface = new RepositoryInterface(); + List resultSet = new ArrayList<>(); - repositoryInterface.setId(repositoryInterfaceObject.get("id").toString()); - repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString()); - repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString()); - - // /* 07-04-2022 | "typology" -> "eoscDatasourceType" */ -// // TODO: enable in future release -// try { // FIXME: remove attemp to get typology if eoscDatasourceType fails -// repositoryInterface.setTypology(repositoryInterfaceObject.get("eoscDatasourceType").toString()); -// } catch (JSONException e) { -// repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString()); -// } - repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString()); -// <-- - - repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString()); - repositoryInterface.setLastCollectionDate(repositoryInterfaceObject.get("lastCollectionDate").toString()); - - repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString()); - repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString())); - - - // repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString()); - repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString()); - //repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString())); - - - Map accessParams = new HashMap<>(); - Map extraFields = new HashMap<>(); - - ObjectMapper mapper = new ObjectMapper(); - JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiParams"); - - for (int i = 0; i < apiparams.length(); i++) - accessParams.put(apiparams.getJSONObject(i).getString("param"), apiparams.getJSONObject(i).getString("value")); - - repositoryInterface.setAccessParams(accessParams); - - return repositoryInterface; + for (ApiDetails entry : apiDetailsList) + resultSet.add(toRepositoryInterface(entry)); + return resultSet; } - public static String repositoryObjectToJson(Repository repository) throws JSONException, JsonProcessingException { + public RepositoryInterface toRepositoryInterface(Object repositoryInterfaceObject) { - HashMap repositoryMap = new HashMap<>(); - ObjectMapper mapper = new ObjectMapper(); + RepositoryInterface repoInterface = objectMapper.convertValue(repositoryInterfaceObject, RepositoryInterface.class); - repositoryMap.put("id", repository.getId()); - repositoryMap.put("openaireId", getOpenaireId(repository.getId())); - repositoryMap.put("officialname", repository.getOfficialName()); - repositoryMap.put("englishname", repository.getEnglishName()); - repositoryMap.put("websiteurl", repository.getWebsiteUrl()); - repositoryMap.put("logourl", repository.getLogoUrl()); - repositoryMap.put("contactemail", repository.getContactEmail()); - repositoryMap.put("longitude", repository.getLongitude().toString()); - repositoryMap.put("latitude", repository.getLatitude().toString()); - repositoryMap.put("timezone", repository.getTimezone()); - - repositoryMap.put("namespaceprefix", repository.getNamespacePrefix() != null ? repository.getNamespacePrefix() : ""); - repositoryMap.put("languages", repository.getOdLanguages() != null ? repository.getOdLanguages() : ""); - - repositoryMap.put("dateofcollection", repository.getDateOfCollection() != null ? convertDateToString(repository.getDateOfCollection()) : ""); - - /* - * typology -> platform - * datasource class -> typology - * */ -// repositoryMap.put("eoscDatasourceType", repository.getDatasourceClass()); // TODO: enable in future release - repositoryMap.put("typology", repository.getDatasourceClass()); - repositoryMap.put("platform", repository.getTypology()); - - repositoryMap.put("dateofvalidation", repository.getDateOfCollection() != null ? convertDateToString(repository.getDateOfCollection()) : ""); - repositoryMap.put("activationId", repository.getActivationId() != null ? repository.getActivationId() : ""); - - repositoryMap.put("description", repository.getDescription()); - - repositoryMap.put("eissn", repository.getEissn() != null ? repository.getEissn() : ""); - repositoryMap.put("issn", repository.getIssn() != null ? repository.getIssn() : ""); - repositoryMap.put("lissn", repository.getLissn() != null ? repository.getLissn() : ""); - - repositoryMap.put("registeredby", repository.getRegisteredBy()); - - repositoryMap.put("aggregator", repository.getAggregator() != null ? repository.getAggregator() : ""); - repositoryMap.put("collectedfrom", repository.getCollectedFrom() != null ? repository.getCollectedFrom() : ""); - - repositoryMap.put("managed", repository.isRegistered()); - - Map organization = new HashMap<>(); - organization.put("legalname", repository.getOrganization()); - organization.put("country", repository.getCountryCode()); - organization.put("legalshortname", ""); - organization.put("websiteurl", ""); - organization.put("logourl", ""); - - List organizations = new ArrayList(); - organizations.add(organization); - repositoryMap.put("organizations", organizations); - - //TODO check identitites - //Map identity = new HashMap<>(); - - if (repository.getPiwikInfo() != null) { - Map identity = new HashMap<>(); - HashSet> identities = new HashSet<>(); - - identity.put("issuertype", "piwik"); - identity.put("pid", "piwik:" + repository.getPiwikInfo().getSiteId()); - - identities.add(identity); - - repositoryMap.put("identities", identities); - } - - repositoryMap.put("subjects", ""); - repositoryMap.put("consentTermsOfUse", repository.getConsentTermsOfUse()); - repositoryMap.put("fullTextDownload", repository.getFullTextDownload()); - repositoryMap.put("consentTermsOfUseDate", convertDateToString(repository.getConsentTermsOfUseDate())); - repositoryMap.put("lastConsentTermsOfUseDate", convertDateToString(repository.getLastConsentTermsOfUseDate())); - - return mapper.writeValueAsString(repositoryMap); + return repoInterface; } - public static String repositoryInterfaceObjectToJson(Repository repository, RepositoryInterface repositoryInterface) throws JSONException { + public String toJson(Repository repository) throws JsonProcessingException { - JSONObject jsonObject = new JSONObject(); - - jsonObject.put("id", repositoryInterface.getId()); - jsonObject.put("protocol", repositoryInterface.getAccessProtocol()); - jsonObject.put("datasource", repository.getId()); - jsonObject.put("contentdescription", repositoryInterface.getContentDescription()); - jsonObject.put("typology", repositoryInterface.getTypology()); - jsonObject.put("compatibility", repositoryInterface.getDesiredCompatibilityLevel()); - jsonObject.put("compatibilityOverride", repositoryInterface.getDesiredCompatibilityLevel()); - - jsonObject.put("lastCollectionTotal", ""); - - jsonObject.put("lastCollectionDate", repositoryInterface.getLastCollectionDate()); - jsonObject.put("lastAggregationTotal", ""); - jsonObject.put("lastAggregationDate", ""); - jsonObject.put("lastDownloadTotal", ""); - jsonObject.put("lastDownloadDate", ""); - - jsonObject.put("baseurl", repositoryInterface.getBaseUrl()); - jsonObject.put("removable", repositoryInterface.isRemovable()); - - - JSONArray apiparams = new JSONArray(); - for (String param : repositoryInterface.getAccessParams().keySet()) { - JSONObject jo = new JSONObject(); - jo.put("param", param); - jo.put("value", repositoryInterface.getAccessParams().get(param)); - apiparams.put(jo); - } - jsonObject.put("apiParams", apiparams); - - -// jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath()); - - - return jsonObject.toString(); + return objectMapper.writeValueAsString(repository); } - public static ArrayList readFile(String filename) { + public List readFile(String filename) { String line; - ArrayList list = new ArrayList(); + List list = new ArrayList<>(); try { //InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename); InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename); @@ -403,32 +99,16 @@ public class Converter { return list; } - public static List getAggregationHistoryFromJson(JSONArray aggregationInfo) throws JSONException { + public List toAggregationHistory(JSONArray aggregationInfo) throws JSONException { List aggregationDetailsList = new ArrayList<>(); for (int i = 0; i < aggregationInfo.length(); i++) - aggregationDetailsList.add(jsonToAggregationDetails(aggregationInfo.getJSONObject(i))); + aggregationDetailsList.add(toAggregationDetails(aggregationInfo.getJSONObject(i))); return aggregationDetailsList; } - private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException { - - AggregationDetails aggregationDetails = new AggregationDetails(); - - if (aggregationObject.has("collectionMode")) - aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString()); - if (aggregationObject.has("indexedVersion")) - aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString())); - - aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString()); - aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString())); - aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString())); - - return aggregationDetails; - } - - public static List toTimezones(List timezones) { + public List toTimezones(List timezones) { List tmz = new ArrayList<>(); for (String t : timezones) { @@ -438,10 +118,60 @@ public class Converter { return tmz; } - private static String getOpenaireId(String repositoryId) { + private String getOpenaireId(String repositoryId) { if (repositoryId != null && repositoryId.contains("::")) return repositoryId.split("::")[0] + "::" + DigestUtils.md5Hex(repositoryId.split("::")[1]); return null; } + private Boolean convertStringToBoolean(String value) { + return value.equals("null") ? null : Boolean.valueOf(value); + } + + private Date toDate(String date) { + + if (Objects.equals(date, "null")) + return null; + + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + try { + return formatter.parse(date); + } catch (ParseException e) { + LOGGER.error(e); + } + return null; + } + + public String toString(Date date) { + + if (Objects.equals(date, null)) + return null; + + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); + return formatter.format(date); + } + + private Double toDouble(String number) { + if (Objects.equals(number, "null")) + return 0.0; + else + return Double.valueOf(number); + } + + private AggregationDetails toAggregationDetails(JSONObject aggregationObject) throws JSONException { + + AggregationDetails aggregationDetails = new AggregationDetails(); + + if (aggregationObject.has("collectionMode")) + aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString()); + if (aggregationObject.has("indexedVersion")) + aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString())); + + aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString()); + aggregationDetails.setDate(toDate(aggregationObject.get("date").toString())); + aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString())); + + return aggregationDetails; + } } +