MaDgIK
/
uoa-repository-manager-service
13
2
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

replaced AggregationDetails with AggregationInfo class

This commit is contained in:
Konstantinos Spyrou 2022-11-30 20:58:08 +02:00
parent a792fd34c1
commit b3758da868
10 changed files with 140 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
pom.xml
View File

@ -74,6 +74,13 @@
<version>3.0.0</version>
</dependency>
-->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-exporter-api</artifactId>
<version>[3.3.3-SNAPSHOT, )</version>
</dependency>
<dependency>
<groupId>eu.dnetlib</groupId>
<artifactId>uoa-clients</artifactId>
@ -509,30 +516,6 @@
<enabled>false</enabled>
</releases>
</repository>
<repository>
<id>dnet45-bootstrap-snapshot</id>
<name>D-Net 45 Bootstrap Snapshot</name>
<url>https://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-bootstrap-snapshot</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
<layout>default</layout>
</repository>
<repository>
<id>dnet45-bootstrap-release</id>
<name>D-Net 45 Bootstrap Release</name>
<url>https://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-bootstrap-release</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<layout>default</layout>
</repository>
</repositories>
</project>

7
src/main/java/eu/dnetlib/repo/manager/controllers/DashboardController.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.repo.manager.controllers;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.repo.manager.domain.*;
import eu.dnetlib.repo.manager.exception.BrokerException;
import eu.dnetlib.repo.manager.exception.RepositoryServiceException;
@ -53,14 +54,14 @@ public class DashboardController {
@PathVariable("repoId") String repoId,
@RequestParam(name = "size", required = false, defaultValue = "20") int size) throws JSONException {
List<AggregationDetails> aggregationDetails = repositoryService.getRepositoryAggregations(repoId, 0, size);
List<AggregationInfo> aggregationDetails = repositoryService.getRepositoryAggregations(repoId, 0, size);
CollectionMonitorSummary collectionMonitorSummary = new CollectionMonitorSummary();
collectionMonitorSummary.setAggregationDetails(aggregationDetails);
size = 0;
do {
aggregationDetails = repositoryService.getRepositoryAggregations(repoId, size, size + 50);
for (AggregationDetails aggregationDetail : aggregationDetails) {
if (aggregationDetail.getIndexedVersion()) {
for (AggregationInfo aggregationDetail : aggregationDetails) {
if (aggregationDetail.isIndexedVersion()) {
collectionMonitorSummary.setLastIndexedVersion(aggregationDetail);
break;
}

5
src/main/java/eu/dnetlib/repo/manager/controllers/RepositoryController.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.repo.manager.controllers;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.repo.manager.domain.Repository;
import eu.dnetlib.repo.manager.domain.RepositoryInterface;
import eu.dnetlib.repo.manager.domain.*;
@ -121,14 +122,14 @@ public class RepositoryController {
@RequestMapping(value = "/getRepositoryAggregations/{id}", method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseBody
public List<AggregationDetails> getRepositoryAggregations(@PathVariable("id") String id) throws JSONException {
public List<AggregationInfo> getRepositoryAggregations(@PathVariable("id") String id) throws JSONException {
return repositoryService.getRepositoryAggregations(id, 0, 20);
}
@RequestMapping(value = "/getRepositoryAggregationsByYear/{id}", method = RequestMethod.GET,
produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseBody
public Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(@PathVariable("id") String id) throws JSONException {
public Map<String, List<AggregationInfo>> getRepositoryAggregationsByYear(@PathVariable("id") String id) throws JSONException {
return repositoryService.getRepositoryAggregationsByYear(id);
}

30
src/main/java/eu/dnetlib/repo/manager/domain/AggregationHistoryResponse.java Normal file
View File

@ -0,0 +1,30 @@
package eu.dnetlib.repo.manager.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import java.util.List;
@JsonAutoDetect
public class AggregationHistoryResponse extends Response {
private List<AggregationInfo> aggregationInfo;
public AggregationHistoryResponse() {
// no-arg constructor
}
public AggregationHistoryResponse(final List<AggregationInfo> aggregationInfo) {
super();
this.aggregationInfo = aggregationInfo;
}
public List<AggregationInfo> getAggregationInfo() {
return aggregationInfo;
}
public void setAggregationInfo(final List<AggregationInfo> aggregationInfo) {
this.aggregationInfo = aggregationInfo;
}
}

16
src/main/java/eu/dnetlib/repo/manager/domain/CollectionMonitorSummary.java
View File

@ -1,32 +1,34 @@
package eu.dnetlib.repo.manager.domain;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import java.util.List;
public class CollectionMonitorSummary {
private List<AggregationDetails> aggregationDetails;
private List<AggregationInfo> aggregationDetails;
private AggregationDetails lastIndexedVersion;
private AggregationInfo lastIndexedVersion;
public CollectionMonitorSummary(){}
public CollectionMonitorSummary(List<AggregationDetails> aggregationDetails, AggregationDetails lastIndexedVersion) {
public CollectionMonitorSummary(List<AggregationInfo> aggregationDetails, AggregationInfo lastIndexedVersion) {
this.aggregationDetails = aggregationDetails;
this.lastIndexedVersion = lastIndexedVersion;
}
public List<AggregationDetails> getAggregationDetails() {
public List<AggregationInfo> getAggregationDetails() {
return aggregationDetails;
}
public void setAggregationDetails(List<AggregationDetails> aggregationDetails) {
public void setAggregationDetails(List<AggregationInfo> aggregationDetails) {
this.aggregationDetails = aggregationDetails;
}
public AggregationDetails getLastIndexedVersion() {
public AggregationInfo getLastIndexedVersion() {
return lastIndexedVersion;
}
public void setLastIndexedVersion(AggregationDetails lastIndexedVersion) {
public void setLastIndexedVersion(AggregationInfo lastIndexedVersion) {
this.lastIndexedVersion = lastIndexedVersion;
}
}

10
src/main/java/eu/dnetlib/repo/manager/service/DashboardServiceImpl.java
View File

@ -1,9 +1,11 @@
package eu.dnetlib.repo.manager.service;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.repo.manager.domain.*;
import eu.dnetlib.repo.manager.domain.broker.BrowseEntry;
import eu.dnetlib.repo.manager.exception.BrokerException;
import eu.dnetlib.repo.manager.exception.RepositoryServiceException;
import eu.dnetlib.repo.manager.utils.DateUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -40,11 +42,11 @@ public class DashboardServiceImpl implements DashboardService {
//TODO getRepositoryAggregations returns only the 20 more recent items. Is it positive that we will find an indexed version there?
long start = System.currentTimeMillis();
List<AggregationDetails> aggregationDetailsList = repositoryService.getRepositoryAggregations(repository.getId(), 0, 20);
for (AggregationDetails aggregationDetails : aggregationDetailsList) {
if (aggregationDetails.getIndexedVersion()) {
List<AggregationInfo> aggregationDetailsList = repositoryService.getRepositoryAggregations(repository.getId(), 0, 20);
for (AggregationInfo aggregationDetails : aggregationDetailsList) {
if (aggregationDetails.isIndexedVersion()) {
repositorySummaryInfo.setRecordsCollected(aggregationDetails.getNumberOfRecords());
repositorySummaryInfo.setLastIndexedVersion(aggregationDetails.getDate());
repositorySummaryInfo.setLastIndexedVersion(DateUtils.toDate(aggregationDetails.getDate()));
break;
}
}

9
src/main/java/eu/dnetlib/repo/manager/service/RepositoryService.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.repo.manager.service;
import eu.dnetlib.repo.manager.domain.Repository;
import eu.dnetlib.repo.manager.domain.RepositoryInterface;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.repo.manager.domain.*;
import eu.dnetlib.repo.manager.exception.RepositoryServiceException;
import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;
@ -44,11 +43,11 @@ public interface RepositoryService {
Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException;
List<AggregationDetails> getRepositoryAggregations(String id) throws JSONException;
List<AggregationInfo> getRepositoryAggregations(String id) throws JSONException;
List<AggregationDetails> getRepositoryAggregations(String id, int from, int size) throws JSONException;
List<AggregationInfo> getRepositoryAggregations(String id, int from, int size) throws JSONException;
Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(String id) throws JSONException;
Map<String, List<AggregationInfo>> getRepositoryAggregationsByYear(String id) throws JSONException;
List<Repository> getRepositoriesByName(String name,
String page,

43
src/main/java/eu/dnetlib/repo/manager/service/RepositoryServiceImpl.java
View File

@ -5,6 +5,7 @@ import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import eu.dnetlib.api.functionality.ValidatorServiceException;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.repo.manager.domain.Repository;
import eu.dnetlib.repo.manager.domain.RepositoryInterface;
import eu.dnetlib.domain.enabling.Vocabulary;
@ -20,6 +21,7 @@ import eu.dnetlib.repo.manager.service.security.AuthoritiesUpdater;
import eu.dnetlib.repo.manager.service.security.AuthorizationService;
import eu.dnetlib.repo.manager.service.security.RoleMappingService;
import eu.dnetlib.repo.manager.utils.Converter;
import eu.dnetlib.repo.manager.utils.DateUtils;
import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.log4j.Logger;
@ -48,6 +50,8 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static eu.dnetlib.repo.manager.utils.DateUtils.getYear;
@Service("repositoryService")
public class RepositoryServiceImpl implements RepositoryService {
@ -410,45 +414,42 @@ public class RepositoryServiceImpl implements RepositoryService {
@Override
public List<AggregationDetails> getRepositoryAggregations(String id) throws JSONException {
public List<AggregationInfo> getRepositoryAggregations(String id) {
LOGGER.debug("Retrieving aggregations for repository with id : " + id);
UriComponents uriComponents = getAggregationHistory(id);
String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
JSONArray aggregationInfo = new JSONObject(rs).getJSONArray("aggregationInfo");
AggregationHistoryResponse rs = restTemplate.getForObject(uriComponents.toUri(), AggregationHistoryResponse.class);
List<AggregationDetails> aggregationHistory = new ArrayList<>(converter.toAggregationHistory(aggregationInfo));
return aggregationHistory;
return rs != null ? rs.getAggregationInfo() : null;
}
@Override
public List<AggregationDetails> getRepositoryAggregations(String id, int from, int size) throws JSONException {
public List<AggregationInfo> getRepositoryAggregations(String id, int from, int size) {
List<AggregationDetails> res = getRepositoryAggregations(id);
List<AggregationInfo> res = getRepositoryAggregations(id);
return res.subList(from, Math.min(from + size, res.size()));
return res.subList(from, Math.min(from + size, res.size()));
}
@Override
public Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(String id) throws JSONException {
public Map<String, List<AggregationInfo>> getRepositoryAggregationsByYear(String id) {
LOGGER.debug("Retrieving aggregations (by year) for repository with id : " + id);
List<AggregationDetails> aggregationHistory = getRepositoryAggregations(id);
Map<String, List<AggregationDetails>> aggregationByYear = new HashMap<>();
List<AggregationInfo> aggregationHistory = getRepositoryAggregations(id);
Map<String, List<AggregationInfo>> aggregationByYear = new HashMap<>();
return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory);
}
private Map<String, List<AggregationDetails>> createYearMap(List<AggregationDetails> aggregationHistory) {
private Map<String, List<AggregationInfo>> createYearMap(List<AggregationInfo> aggregationHistory) {
aggregationHistory = aggregationHistory.stream()
.sorted(Comparator.comparing(AggregationDetails::getDate).reversed())
.sorted(Comparator.comparing(AggregationInfo::getDate).reversed())
.collect(Collectors.toList());
return aggregationHistory.stream()
.collect(Collectors.groupingBy(AggregationDetails::getYear));
.collect(Collectors.groupingBy(item -> getYear(item.getDate())));
}
@ -925,15 +926,15 @@ public class RepositoryServiceImpl implements RepositoryService {
public Map<String, String> getListLatestUpdate(String mode) throws JSONException {
Map<String, String> dates = new HashMap<>();
if (mode.equals("repository")) {
dates.put("opendoar", converter.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate()));
dates.put("fairsharing", converter.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate()));
dates.put("opendoar", DateUtils.toString(getRepositoryInterface("openaire____::opendoar").get(0).getLastCollectionDate()));
dates.put("fairsharing", DateUtils.toString(getRepositoryInterface("openaire____::fairsharing").get(0).getLastCollectionDate()));
// create re3data last collection date
// dates.put("re3data", converter.toString(getRepositoryInterface("openaire____::re3data").get(1).getLastCollectionDate()));
List<RepositoryInterface> re3interfaces = getRepositoryInterface("openaire____::re3data");
String re3Date = null;
for (RepositoryInterface interf : re3interfaces) {
if (interf.getLastCollectionDate() != null) {
re3Date = converter.toString(interf.getLastCollectionDate());
re3Date = DateUtils.toString(interf.getLastCollectionDate());
break;
}
}
@ -942,14 +943,14 @@ public class RepositoryServiceImpl implements RepositoryService {
return dates;
}
else if (mode.equals("cris"))
return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate()));
return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("eurocrisdris::dris").get(0).getLastCollectionDate()));
else if (mode.equals("opendoar")) // TODO: remove this and else clause
return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
else
/*
* first api of re3data has null value on collection date
* */
return Collections.singletonMap("lastCollectionDate", converter.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
return Collections.singletonMap("lastCollectionDate", DateUtils.toString(getRepositoryInterface("openaire____::" + mode).get(0).getLastCollectionDate()));
}
private void updateValidationSet(String repositoryId, String repositoryInterfaceId, String validationSet) throws Exception {

34
src/main/java/eu/dnetlib/repo/manager/utils/Converter.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.repo.manager.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.repo.manager.domain.*;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.log4j.Logger;
@ -99,11 +100,13 @@ public class Converter {
return list;
}
public List<AggregationDetails> toAggregationHistory(JSONArray aggregationInfo) throws JSONException {
List<AggregationDetails> aggregationDetailsList = new ArrayList<>();
public List<AggregationInfo> toAggregationHistory(String aggregationHistoryResponse) throws JSONException {
List<AggregationInfo> aggregationDetailsList = new ArrayList<>();
JSONArray aggregationInfo = new JSONObject(aggregationHistoryResponse).getJSONArray("aggregationInfo");
for (int i = 0; i < aggregationInfo.length(); i++)
aggregationDetailsList.add(toAggregationDetails(aggregationInfo.getJSONObject(i)));
// aggregationDetailsList.add(toAggregationDetails(aggregationInfo.getJSONObject(i)));
aggregationDetailsList.add(objectMapper.convertValue(aggregationInfo.getJSONObject(i), AggregationInfo.class));
return aggregationDetailsList;
}
@ -128,29 +131,6 @@ public class Converter {
return value.equals("null") ? null : Boolean.valueOf(value);
}
private Date toDate(String date) {
if (Objects.equals(date, "null"))
return null;
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
try {
return formatter.parse(date);
} catch (ParseException e) {
LOGGER.error(e);
}
return null;
}
public String toString(Date date) {
if (Objects.equals(date, null))
return null;
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
return formatter.format(date);
}
private Double toDouble(String number) {
if (Objects.equals(number, "null"))
return 0.0;
@ -168,7 +148,7 @@ public class Converter {
aggregationDetails.setIndexedVersion(Boolean.parseBoolean(aggregationObject.get("indexedVersion").toString()));
aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());
aggregationDetails.setDate(toDate(aggregationObject.get("date").toString()));
aggregationDetails.setDate(DateUtils.toDate(aggregationObject.get("date").toString()));
aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));
return aggregationDetails;

48
src/main/java/eu/dnetlib/repo/manager/utils/DateUtils.java Normal file
View File

@ -0,0 +1,48 @@
package eu.dnetlib.repo.manager.utils;
import org.apache.log4j.Logger;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Objects;
public class DateUtils {
private static final Logger logger = Logger.getLogger(DateUtils.class);
public static Date toDate(String date) {
if (Objects.equals(date, "null"))
return null;
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
try {
return formatter.parse(date);
} catch (ParseException e) {
logger.error(e);
}
return null;
}
public static String toString(Date date) {
if (Objects.equals(date, null))
return null;
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
return formatter.format(date);
}
public static String getYear(String date) {
Calendar calendar = Calendar.getInstance();
calendar.setTime(Objects.requireNonNull(toDate(date)));
return String.valueOf(calendar.get(Calendar.YEAR));
}
private DateUtils() {
}
}