From a61b259ca3f4245059063f3f86c51320a442867c Mon Sep 17 00:00:00 2001 From: Antonis Lempesis Date: Tue, 24 May 2022 14:01:08 +0000 Subject: [PATCH] getting aggregations uising the correct dsm api method --- .../manager/service/RepositoryService.java | 2 + .../service/RepositoryServiceImpl.java | 104 ++++++------------ .../dnetlib/repo/manager/utils/Converter.java | 16 +-- 3 files changed, 44 insertions(+), 78 deletions(-) diff --git a/src/main/java/eu/dnetlib/repo/manager/service/RepositoryService.java b/src/main/java/eu/dnetlib/repo/manager/service/RepositoryService.java index bd1636a..76a9435 100644 --- a/src/main/java/eu/dnetlib/repo/manager/service/RepositoryService.java +++ b/src/main/java/eu/dnetlib/repo/manager/service/RepositoryService.java @@ -43,6 +43,8 @@ public interface RepositoryService { Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException; + List getRepositoryAggregations(String id) throws JSONException; + List getRepositoryAggregations(String id, int from, int size) throws JSONException; Map> getRepositoryAggregationsByYear(String id) throws JSONException; diff --git a/src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java b/src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java index 4a65723..f8904c6 100644 --- a/src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java +++ b/src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java @@ -105,12 +105,6 @@ public class RepositoryServiceImpl implements RepositoryService { this.restTemplate = restTemplate; } - private String getAuthenticatedUserEmail() { - OIDCAuthenticationToken authenticationToken = (OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication(); - return authenticationToken.getUserInfo().getEmail(); - } - - @PostConstruct private void init() { LOGGER.debug("Initialization method of repository api!"); @@ -130,8 +124,6 @@ public class RepositoryServiceImpl implements RepositoryService { dataSourceClass.get("opendoar").add(key); } else if (key.contains("datarepository")) { dataSourceClass.putIfAbsent("re3data", Collections.singletonList("datarepository::unknown")); - } else if (key.contains("pubsrepository::journal")) { - dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal")); } } @@ -177,7 +169,7 @@ public class RepositoryServiceImpl implements RepositoryService { @Override public List getRepositories(List ids, int page, int size) throws JSONException { List repos = new ArrayList<>(); - LOGGER.debug("Retreiving repositories with ids : " + String.join(", ", ids)); + LOGGER.debug("Retrieving repositories with ids : " + String.join(", ", ids)); UriComponents uriComponents = searchDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size))); RequestFilter requestFilter = new RequestFilter(); @@ -331,29 +323,17 @@ LOGGER.debug("json: " + jsonArray); return r; } - - private Collection getRepositoriesByMode(String mode, List rs) { - - List reps = new ArrayList<>(); - for (Repository r : rs) { - if (r.getCollectedFrom() != null && r.getCollectedFrom().equals(mode)) - reps.add(r); - - } - return reps; - } - @Override public List getRepositoriesOfUser(String page, String size) throws JSONException { String userEmail = ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail(); - LOGGER.debug("Retreiving repositories of authenticated user : " + userEmail); + LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail); Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles()); return getRepositories(new ArrayList<>(repoIds)); } @Override public List getRepositoriesOfUser(String userEmail, String page, String size) throws JSONException { - LOGGER.debug("Retreiving repositories of authenticated user : " + userEmail); + LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail); Collection repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail)); return getRepositories(new ArrayList<>(repoIds)); } @@ -373,7 +353,7 @@ LOGGER.debug("json: " + jsonArray); @Override public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException { - LOGGER.debug("Retreiving repositories with id : " + id); + LOGGER.debug("Retrieving repositories with id : " + id); RepositorySnippet repo = null; UriComponents uriComponents = searchSnipperDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); @@ -392,7 +372,7 @@ LOGGER.debug("json: " + jsonArray); @Override public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException { - LOGGER.debug("Retreiving repositories with id : " + id); + LOGGER.debug("Retrieving repositories with id : " + id); Repository repo = null; UriComponents uriComponents = searchDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); @@ -409,60 +389,41 @@ LOGGER.debug("json: " + jsonArray); } + @Override + public List getRepositoryAggregations(String id) throws JSONException { + + LOGGER.debug("Retrieving aggregations for repository with id : " + id); + UriComponents uriComponents = getAggregationHistory(id); + + String rs = restTemplate.getForObject(uriComponents.toUri(), String.class); + JSONArray aggregationInfo = new JSONObject(rs).getJSONArray("aggregationInfo"); + + List aggregationHistory = new ArrayList<>(Converter.getAggregationHistoryFromJson(aggregationInfo)); + + return aggregationHistory; +// return aggregationHistory.size() == 0 ? aggregationHistory : aggregationHistory.stream() +// .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) +// .collect(Collectors.toList()); + } + @Override public List getRepositoryAggregations(String id, int from, int size) throws JSONException { - LOGGER.debug("Retreiving aggregations for repository with id : " + id); - UriComponents uriComponents = searchDatasource(from + "", size + ""); - RequestFilter requestFilter = new RequestFilter(); - requestFilter.setId(id); - - List aggregationHistory = new ArrayList<>(); - - long start = System.currentTimeMillis(); - String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); - long end = System.currentTimeMillis(); - - System.out.println("Aggregations request through rest template took " + (end - start) + "ms"); - JSONObject repository = new JSONObject(rs); - - if (repository.getJSONArray("datasourceInfo").length() == 0) - return aggregationHistory; - - start = System.currentTimeMillis(); - aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0))); - end = System.currentTimeMillis(); - - System.out.println("Getting aggregations history from json " + (end - start) + "ms"); - return aggregationHistory.size() == 0 ? aggregationHistory : aggregationHistory.stream() - .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) - .limit(size) - .collect(Collectors.toList()); - + return getRepositoryAggregations(id).subList(from, from + size); } @Override public Map> getRepositoryAggregationsByYear(String id) throws JSONException { - LOGGER.debug("Retreiving aggregations (by year) for repository with id : " + id); - UriComponents uriComponents = searchDatasource("0", "100"); - RequestFilter requestFilter = new RequestFilter(); - requestFilter.setId(id); + LOGGER.debug("Retrieving aggregations (by year) for repository with id : " + id); - List aggregationHistory = new ArrayList<>(); + List aggregationHistory = getRepositoryAggregations(id); Map> aggregationByYear = new HashMap<>(); - - String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class); - JSONObject repository = new JSONObject(rs); - - if (repository.getJSONArray("datasourceInfo").length() == 0) - return aggregationByYear; - - aggregationHistory.addAll(Converter.getAggregationHistoryFromJson(repository.getJSONArray("datasourceInfo").getJSONObject(0))); + return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory); } private Map> createYearMap(List aggregationHistory) { - Map> aggregationByYear; + aggregationHistory = aggregationHistory.stream() .sorted(Comparator.comparing(AggregationDetails::getDate).reversed()) .collect(Collectors.toList()); @@ -477,7 +438,7 @@ LOGGER.debug("json: " + jsonArray); String page, String size) throws JSONException { - LOGGER.debug("Retreiving repositories with official name : " + name); + LOGGER.debug("Retrieving repositories with official name : " + name); UriComponents uriComponents = searchDatasource("0", "100"); RequestFilter requestFilter = new RequestFilter(); requestFilter.setOfficialname(name); @@ -571,7 +532,7 @@ LOGGER.debug("json: " + jsonArray); String json_repository = Converter.repositoryObjectToJson(repository); LOGGER.debug("JSON to add(update) -> " + json_repository); - HttpEntity httpEntity = new HttpEntity(json_repository, httpHeaders); + HttpEntity httpEntity = new HttpEntity<>(json_repository, httpHeaders); ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class); if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { @@ -1000,6 +961,13 @@ LOGGER.debug("json: " + jsonArray); return null; } + private UriComponents getAggregationHistory(String repoId) { + return UriComponentsBuilder + .fromHttpUrl(baseAddress + "/ds/aggregationhistory/") + .path("/repoId") + .build().expand(repoId).encode(); + } + private UriComponents searchDatasource(String page, String size) { return UriComponentsBuilder diff --git a/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java b/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java index 73a8d40..bfeb8e0 100644 --- a/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java +++ b/src/main/java/eu/dnetlib/repo/manager/utils/Converter.java @@ -402,16 +402,11 @@ public class Converter { return list; } - public static List getAggregationHistoryFromJson(JSONObject datasourceInfo) throws JSONException { - JSONArray rs; + public static List getAggregationHistoryFromJson(JSONArray aggregationInfo) throws JSONException { List aggregationDetailsList = new ArrayList<>(); - if (datasourceInfo.get("aggregationHistory") != null && !datasourceInfo.get("aggregationHistory").toString().equals("null")) { - rs = new JSONArray(datasourceInfo.get("aggregationHistory").toString()); - - for (int i = 0; i < rs.length(); i++) - aggregationDetailsList.add(jsonToAggregationDetails(rs.getJSONObject(i))); - } + for (int i = 0; i < aggregationInfo.length(); i++) + aggregationDetailsList.add(jsonToAggregationDetails(aggregationInfo.getJSONObject(i))); return aggregationDetailsList; } @@ -420,17 +415,18 @@ public class Converter { AggregationDetails aggregationDetails = new AggregationDetails(); - aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString()); if (aggregationObject.has("collectionMode")) aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString()); if (aggregationObject.has("indexedVersion")) aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString())); + + aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString()); aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString())); aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString())); + return aggregationDetails; } - public static List toTimezones(List timezones) { List tmz = new ArrayList<>();