merged the sub classes in the main class

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java

@@ -48,11 +48,7 @@ import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfoV1;
 import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV2;
 import eu.dnetlib.openaire.exporter.model.dsm.CollectionMode;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV1;
-import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV2;
 import eu.dnetlib.openaire.info.JdbcInfoDao;
 
 /**
@@ -75,16 +71,16 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private final static String LOADTIME = "loadtime";
 	private final LoadingCache<String, Instant> loadingCache =
-		CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
+			CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
 
-			// The only cached value is associated to "loadtime"
+				// The only cached value is associated to "loadtime"
-			@Override
+				@Override
-			public Instant load(final String key) {
+				public Instant load(final String key) {
-				final Instant loadTime = getLoadTime();
+					final Instant loadTime = getLoadTime();
-				log.debug("found load time: " + loadTime.toString());
+					log.debug("found load time: " + loadTime.toString());
-				return loadTime;
+					return loadTime;
-			}
+				}
-		});
+			});
 
 	private static final Bson fields = getFields();
 
@@ -94,30 +90,64 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	@Cacheable("dsm-aggregationhistory-cache-v1")
 	@Deprecated
 	public List<AggregationInfoV1> getAggregationHistoryV1(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
+		return getAggregationHistoryV1(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
 	}
 
 	@Override
 	@Cacheable("dsm-aggregationhistory-cache-v2")
 	public List<AggregationInfo> getAggregationHistoryV2(final String dsId) throws DsmApiException {
-		return getAggregationHistory(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
+		return getAggregationHistoryV2(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
 	}
 
-	private <T extends AggregationInfo> List<T> getAggregationHistory(final String dsId,
+	@Deprecated
-		final Bson queryForAggregationHistory,
+	private List<AggregationInfoV1> getAggregationHistoryV1(final String dsId,
-		final Function<Document, T> mapper) throws DsmApiException {
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfoV1> mapper) throws DsmApiException {
 		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
 		final Datasource conf = config.getDatasource();
 		try {
 			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
-				.projection(fields)
+					.projection(fields)
-				.limit(conf.getMongoQueryLimit())
+					.limit(conf.getMongoQueryLimit())
-				.sort(dbo("system:startHumanDate", -1));
+					.sort(dbo("system:startHumanDate", -1));
 
-			final List<T> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+			final List<AggregationInfoV1> aggregationInfos = Utils.stream(aggregationDocs.iterator())
-				.map(mapper)
+					.map(mapper)
-				.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
-				.collect(Collectors.toList());
+					.collect(Collectors.toList());
+
+			final Instant loadTime = loadingCache.get(LOADTIME);
+
+			if (!Objects.equals(Instant.MIN, loadTime)) {
+				for (final AggregationInfoV1 a : aggregationInfos) {
+					if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) {
+						a.setIndexedVersion(true);
+						break;
+					}
+				}
+			}
+
+			return aggregationInfos;
+		} catch (final Throwable e) {
+			throw new DsmApiException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
+		}
+	}
+
+	private List<AggregationInfo> getAggregationHistoryV2(final String dsId,
+			final Bson queryForAggregationHistory,
+			final Function<Document, AggregationInfo> mapper) throws DsmApiException {
+		log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
+		final Datasource conf = config.getDatasource();
+		try {
+			final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
+					.projection(fields)
+					.limit(conf.getMongoQueryLimit())
+					.sort(dbo("system:startHumanDate", -1));
+
+			final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
+					.map(mapper)
+					.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
+					.collect(Collectors.toList());
 
 			final Instant loadTime = loadingCache.get(LOADTIME);
 
@@ -146,9 +176,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 		return Instant.parse(a.getDate() + "T00:00:00Z");
 	}
 
+	@Deprecated
+	private Instant asInstant(final AggregationInfoV1 a) {
+		return Instant.parse(a.getDate() + "T00:00:00Z");
+	}
+
 	@Override
 	@CacheEvict(cacheNames = {
-		"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
+			"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
 	}, allEntries = true)
 	@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
 	public void dropCache() {
@@ -166,20 +201,18 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV1 cInfo = new CollectionInfoV1();
+				final AggregationInfoV1 cInfo = new AggregationInfoV1();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				cInfo.setDate(getDate(d));
-				cInfo.setCompletedSuccessfully(success);
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV1 tInfo = new TransformationInfoV1();
+				final AggregationInfoV1 tInfo = new AggregationInfoV1();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
-				tInfo.setCompletedSuccessfully(success);
 				info = tInfo;
 				break;
 			}
@@ -197,7 +230,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 			switch (stage) {
 
 			case COLLECT:
-				final CollectionInfoV2 cInfo = new CollectionInfoV2();
+				final AggregationInfo cInfo = new AggregationInfo();
 				cInfo.setAggregationStage(stage);
 				cInfo.setCollectionMode(getCollectionMode(d));
 				cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
@@ -206,7 +239,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 				info = cInfo;
 				break;
 			case TRANSFORM:
-				final TransformationInfoV2 tInfo = new TransformationInfoV2();
+				final AggregationInfo tInfo = new AggregationInfo();
 				tInfo.setAggregationStage(stage);
 				tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
 				tInfo.setDate(getDate(d));
@@ -220,23 +253,19 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 	private CollectionMode getCollectionMode(final Document d) {
 		return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
-			.map(CollectionMode::valueOf)
-			.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
 				.map(CollectionMode::valueOf)
-				.orElse(null));
+				.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
+						.map(CollectionMode::valueOf)
+						.orElse(null));
 	}
 
 	private Integer getNumberOfRecords(final Document d) {
 		final String sinkSize = d.getString("mainlog:sinkSize");
 		final String total = d.getString("mainlog:total");
 
-		if (StringUtils.isNotBlank(sinkSize)) {
+		if (StringUtils.isNotBlank(sinkSize)) { return Ints.tryParse(sinkSize); }
-			return Ints.tryParse(sinkSize);
+		if (StringUtils.isNotBlank(total)) { return Ints.tryParse(total); }
-		} else if (StringUtils.isNotBlank(total)) {
+		return -1;
-			return Ints.tryParse(total);
-		} else {
-			return -1;
-		}
 	}
 
 	private String getDate(final Document d) {

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java

@@ -9,6 +9,10 @@ public class AggregationHistoryResponseV2 extends Response {
 
 	private List<AggregationInfo> aggregationInfo;
 
+	public AggregationHistoryResponseV2() {
+		super();
+	}
+
 	public AggregationHistoryResponseV2(final List<AggregationInfo> aggregationInfo) {
 		super();
 		this.aggregationInfo = aggregationInfo;

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java

@@ -1,6 +1,16 @@
 package eu.dnetlib.openaire.exporter.model.dsm;
 
-public abstract class AggregationInfo {
+import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+
+@JsonAutoDetect
+@JsonInclude(Include.NON_NULL)
+public class AggregationInfo implements Serializable {
+
+	private static final long serialVersionUID = -4908395195618212510L;
 
 	private int numberOfRecords;
 
@@ -12,6 +22,8 @@ public abstract class AggregationInfo {
 
 	private boolean completedSuccessfully = true;
 
+	private CollectionMode collectionMode;
+
 	public AggregationInfo() {}
 
 	public int getNumberOfRecords() {
@@ -53,4 +65,13 @@ public abstract class AggregationInfo {
 	public void setCompletedSuccessfully(final boolean completedSuccessfully) {
 		this.completedSuccessfully = completedSuccessfully;
 	}
+
+	public CollectionMode getCollectionMode() {
+		return collectionMode;
+	}
+
+	public void setCollectionMode(final CollectionMode collectionMode) {
+		this.collectionMode = collectionMode;
+	}
+
 }

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java

@@ -1,13 +1,67 @@
 package eu.dnetlib.openaire.exporter.model.dsm;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
+import java.io.Serializable;
 
-public class AggregationInfoV1 extends AggregationInfo {
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
 
-	@Override
+@JsonAutoDetect
-	@JsonIgnore
+@JsonInclude(Include.NON_NULL)
-	public boolean isCompletedSuccessfully() {
+public class AggregationInfoV1 implements Serializable {
-		return super.isCompletedSuccessfully();
+
+	private static final long serialVersionUID = 7333873121568523946L;
+
+	private int numberOfRecords;
+
+	private String date;
+
+	private AggregationStage aggregationStage;
+
+	private boolean indexedVersion = false;
+
+	private CollectionMode collectionMode;
+
+	public AggregationInfoV1() {}
+
+	public int getNumberOfRecords() {
+		return numberOfRecords;
+	}
+
+	public void setNumberOfRecords(final int numberOfRecords) {
+		this.numberOfRecords = numberOfRecords;
+	}
+
+	public String getDate() {
+		return date;
+	}
+
+	public void setDate(final String date) {
+		this.date = date;
+	}
+
+	public AggregationStage getAggregationStage() {
+		return aggregationStage;
+	}
+
+	public void setAggregationStage(final AggregationStage aggregationStage) {
+		this.aggregationStage = aggregationStage;
+	}
+
+	public boolean isIndexedVersion() {
+		return indexedVersion;
+	}
+
+	public void setIndexedVersion(final boolean indexedVersion) {
+		this.indexedVersion = indexedVersion;
+	}
+
+	public CollectionMode getCollectionMode() {
+		return collectionMode;
+	}
+
+	public void setCollectionMode(final CollectionMode collectionMode) {
+		this.collectionMode = collectionMode;
 	}
 
 }

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java

@@ -1,22 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-@Deprecated
-public class CollectionInfoV1 extends AggregationInfoV1 {
-
-	private CollectionMode collectionMode;
-
-	public CollectionMode getCollectionMode() {
-		return collectionMode;
-	}
-
-	public void setCollectionMode(final CollectionMode collectionMode) {
-		this.collectionMode = collectionMode;
-	}
-
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java

@@ -1,21 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-public class CollectionInfoV2 extends AggregationInfo {
-
-	private CollectionMode collectionMode;
-
-	public CollectionMode getCollectionMode() {
-		return collectionMode;
-	}
-
-	public void setCollectionMode(final CollectionMode collectionMode) {
-		this.collectionMode = collectionMode;
-	}
-
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java

@@ -1,12 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-@Deprecated
-public class TransformationInfoV1 extends AggregationInfoV1 {
-
-}

libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java

@@ -1,11 +0,0 @@
-package eu.dnetlib.openaire.exporter.model.dsm;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-
-/**
- * Created by claudio on 29/11/2016.
- */
-@JsonAutoDetect
-public class TransformationInfoV2 extends AggregationInfo {
-
-}

libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.openaire.exporter.model.dsm;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class DatasourceApiClientTest {
+
+	@Test
+	public void testaAgregationHistoryV2_local() throws Exception {
+
+		final ObjectMapper mapper = new ObjectMapper();
+
+		final AggregationHistoryResponseV2 res =
+				mapper.readValue(IOUtils.toString(getClass().getResourceAsStream("aggregation-info-v2.json"), "UTF-8"), AggregationHistoryResponseV2.class);
+
+		System.out.println(mapper.writeValueAsString(res));
+
+		assertTrue(res.getAggregationInfo().size() > 0);
+	}
+
+}

libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json

@@ -0,0 +1,27 @@
+{
+  "header": {
+    "total": 100,
+    "page": 0,
+    "size": 100,
+    "time": 0,
+    "statusCode": 0,
+    "errors": []
+  },
+  "aggregationInfo": [
+    {
+      "numberOfRecords": 739,
+      "date": "2024-03-14",
+      "aggregationStage": "TRANSFORM",
+      "indexedVersion": false,
+      "completedSuccessfully": true
+    },
+    {
+      "numberOfRecords": 822,
+      "date": "2024-03-14",
+      "aggregationStage": "COLLECT",
+      "indexedVersion": false,
+      "completedSuccessfully": true,
+      "collectionMode": "REFRESH"
+    }
+  ]
+}