From ed73d524d8a544f3c10d1694aeb2e5580e1455bc Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 18 Mar 2024 11:19:16 +0100 Subject: [PATCH] merged the sub classes in the main class --- .../dsm/dao/MongoLoggerClientImpl.java | 113 +++++++++++------- .../dsm/AggregationHistoryResponseV2.java | 4 + .../exporter/model/dsm/AggregationInfo.java | 23 +++- .../exporter/model/dsm/AggregationInfoV1.java | 66 +++++++++- .../exporter/model/dsm/CollectionInfoV1.java | 22 ---- .../exporter/model/dsm/CollectionInfoV2.java | 21 ---- .../model/dsm/TransformationInfoV1.java | 12 -- .../model/dsm/TransformationInfoV2.java | 11 -- .../model/dsm/DatasourceApiClientTest.java | 25 ++++ .../model/dsm/aggregation-info-v2.json | 27 +++++ 10 files changed, 209 insertions(+), 115 deletions(-) delete mode 100644 libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java delete mode 100644 libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java delete mode 100644 libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java delete mode 100644 libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java create mode 100644 libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java create mode 100644 libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java index 3e3be8f2..008d0bf4 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java @@ -48,11 +48,7 @@ import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfoV1; import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; -import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV1; -import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV2; import eu.dnetlib.openaire.exporter.model.dsm.CollectionMode; -import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV1; -import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV2; import eu.dnetlib.openaire.info.JdbcInfoDao; /** @@ -75,16 +71,16 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { private final static String LOADTIME = "loadtime"; private final LoadingCache loadingCache = - CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader() { + CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader() { - // The only cached value is associated to "loadtime" - @Override - public Instant load(final String key) { - final Instant loadTime = getLoadTime(); - log.debug("found load time: " + loadTime.toString()); - return loadTime; - } - }); + // The only cached value is associated to "loadtime" + @Override + public Instant load(final String key) { + final Instant loadTime = getLoadTime(); + log.debug("found load time: " + loadTime.toString()); + return loadTime; + } + }); private static final Bson fields = getFields(); @@ -94,30 +90,64 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { @Cacheable("dsm-aggregationhistory-cache-v1") @Deprecated public List getAggregationHistoryV1(final String dsId) throws DsmApiException { - return getAggregationHistory(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1()); + return getAggregationHistoryV1(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1()); } @Override @Cacheable("dsm-aggregationhistory-cache-v2") public List getAggregationHistoryV2(final String dsId) throws DsmApiException { - return getAggregationHistory(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2()); + return getAggregationHistoryV2(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2()); } - private List getAggregationHistory(final String dsId, - final Bson queryForAggregationHistory, - final Function mapper) throws DsmApiException { + @Deprecated + private List getAggregationHistoryV1(final String dsId, + final Bson queryForAggregationHistory, + final Function mapper) throws DsmApiException { log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId)); final Datasource conf = config.getDatasource(); try { final FindIterable aggregationDocs = getCollection().find(queryForAggregationHistory) - .projection(fields) - .limit(conf.getMongoQueryLimit()) - .sort(dbo("system:startHumanDate", -1)); + .projection(fields) + .limit(conf.getMongoQueryLimit()) + .sort(dbo("system:startHumanDate", -1)); - final List aggregationInfos = Utils.stream(aggregationDocs.iterator()) - .map(mapper) - .filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate())) - .collect(Collectors.toList()); + final List aggregationInfos = Utils.stream(aggregationDocs.iterator()) + .map(mapper) + .filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate())) + .collect(Collectors.toList()); + + final Instant loadTime = loadingCache.get(LOADTIME); + + if (!Objects.equals(Instant.MIN, loadTime)) { + for (final AggregationInfoV1 a : aggregationInfos) { + if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) { + a.setIndexedVersion(true); + break; + } + } + } + + return aggregationInfos; + } catch (final Throwable e) { + throw new DsmApiException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e); + } + } + + private List getAggregationHistoryV2(final String dsId, + final Bson queryForAggregationHistory, + final Function mapper) throws DsmApiException { + log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId)); + final Datasource conf = config.getDatasource(); + try { + final FindIterable aggregationDocs = getCollection().find(queryForAggregationHistory) + .projection(fields) + .limit(conf.getMongoQueryLimit()) + .sort(dbo("system:startHumanDate", -1)); + + final List aggregationInfos = Utils.stream(aggregationDocs.iterator()) + .map(mapper) + .filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate())) + .collect(Collectors.toList()); final Instant loadTime = loadingCache.get(LOADTIME); @@ -146,9 +176,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { return Instant.parse(a.getDate() + "T00:00:00Z"); } + @Deprecated + private Instant asInstant(final AggregationInfoV1 a) { + return Instant.parse(a.getDate() + "T00:00:00Z"); + } + @Override @CacheEvict(cacheNames = { - "dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache" + "dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache" }, allEntries = true) @Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}") public void dropCache() { @@ -166,20 +201,18 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { switch (stage) { case COLLECT: - final CollectionInfoV1 cInfo = new CollectionInfoV1(); + final AggregationInfoV1 cInfo = new AggregationInfoV1(); cInfo.setAggregationStage(stage); cInfo.setCollectionMode(getCollectionMode(d)); cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0); cInfo.setDate(getDate(d)); - cInfo.setCompletedSuccessfully(success); info = cInfo; break; case TRANSFORM: - final TransformationInfoV1 tInfo = new TransformationInfoV1(); + final AggregationInfoV1 tInfo = new AggregationInfoV1(); tInfo.setAggregationStage(stage); tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0); tInfo.setDate(getDate(d)); - tInfo.setCompletedSuccessfully(success); info = tInfo; break; } @@ -197,7 +230,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { switch (stage) { case COLLECT: - final CollectionInfoV2 cInfo = new CollectionInfoV2(); + final AggregationInfo cInfo = new AggregationInfo(); cInfo.setAggregationStage(stage); cInfo.setCollectionMode(getCollectionMode(d)); cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0); @@ -206,7 +239,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { info = cInfo; break; case TRANSFORM: - final TransformationInfoV2 tInfo = new TransformationInfoV2(); + final AggregationInfo tInfo = new AggregationInfo(); tInfo.setAggregationStage(stage); tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0); tInfo.setDate(getDate(d)); @@ -220,23 +253,19 @@ public class MongoLoggerClientImpl implements MongoLoggerClient { private CollectionMode getCollectionMode(final Document d) { return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection")) - .map(CollectionMode::valueOf) - .orElseGet(() -> Optional.ofNullable(d.getString("collectionMode")) .map(CollectionMode::valueOf) - .orElse(null)); + .orElseGet(() -> Optional.ofNullable(d.getString("collectionMode")) + .map(CollectionMode::valueOf) + .orElse(null)); } private Integer getNumberOfRecords(final Document d) { final String sinkSize = d.getString("mainlog:sinkSize"); final String total = d.getString("mainlog:total"); - if (StringUtils.isNotBlank(sinkSize)) { - return Ints.tryParse(sinkSize); - } else if (StringUtils.isNotBlank(total)) { - return Ints.tryParse(total); - } else { - return -1; - } + if (StringUtils.isNotBlank(sinkSize)) { return Ints.tryParse(sinkSize); } + if (StringUtils.isNotBlank(total)) { return Ints.tryParse(total); } + return -1; } private String getDate(final Document d) { diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java index 1a7779d1..092c6299 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java @@ -9,6 +9,10 @@ public class AggregationHistoryResponseV2 extends Response { private List aggregationInfo; + public AggregationHistoryResponseV2() { + super(); + } + public AggregationHistoryResponseV2(final List aggregationInfo) { super(); this.aggregationInfo = aggregationInfo; diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java index a2d1b7b5..81632ebf 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java @@ -1,6 +1,16 @@ package eu.dnetlib.openaire.exporter.model.dsm; -public abstract class AggregationInfo { +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +@JsonAutoDetect +@JsonInclude(Include.NON_NULL) +public class AggregationInfo implements Serializable { + + private static final long serialVersionUID = -4908395195618212510L; private int numberOfRecords; @@ -12,6 +22,8 @@ public abstract class AggregationInfo { private boolean completedSuccessfully = true; + private CollectionMode collectionMode; + public AggregationInfo() {} public int getNumberOfRecords() { @@ -53,4 +65,13 @@ public abstract class AggregationInfo { public void setCompletedSuccessfully(final boolean completedSuccessfully) { this.completedSuccessfully = completedSuccessfully; } + + public CollectionMode getCollectionMode() { + return collectionMode; + } + + public void setCollectionMode(final CollectionMode collectionMode) { + this.collectionMode = collectionMode; + } + } diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java index 3d70cae7..ccecf763 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java @@ -1,13 +1,67 @@ package eu.dnetlib.openaire.exporter.model.dsm; -import com.fasterxml.jackson.annotation.JsonIgnore; +import java.io.Serializable; -public class AggregationInfoV1 extends AggregationInfo { +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; - @Override - @JsonIgnore - public boolean isCompletedSuccessfully() { - return super.isCompletedSuccessfully(); +@JsonAutoDetect +@JsonInclude(Include.NON_NULL) +public class AggregationInfoV1 implements Serializable { + + private static final long serialVersionUID = 7333873121568523946L; + + private int numberOfRecords; + + private String date; + + private AggregationStage aggregationStage; + + private boolean indexedVersion = false; + + private CollectionMode collectionMode; + + public AggregationInfoV1() {} + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public void setNumberOfRecords(final int numberOfRecords) { + this.numberOfRecords = numberOfRecords; + } + + public String getDate() { + return date; + } + + public void setDate(final String date) { + this.date = date; + } + + public AggregationStage getAggregationStage() { + return aggregationStage; + } + + public void setAggregationStage(final AggregationStage aggregationStage) { + this.aggregationStage = aggregationStage; + } + + public boolean isIndexedVersion() { + return indexedVersion; + } + + public void setIndexedVersion(final boolean indexedVersion) { + this.indexedVersion = indexedVersion; + } + + public CollectionMode getCollectionMode() { + return collectionMode; + } + + public void setCollectionMode(final CollectionMode collectionMode) { + this.collectionMode = collectionMode; } } diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java deleted file mode 100644 index 6c15c896..00000000 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java +++ /dev/null @@ -1,22 +0,0 @@ -package eu.dnetlib.openaire.exporter.model.dsm; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; - -/** - * Created by claudio on 29/11/2016. - */ -@JsonAutoDetect -@Deprecated -public class CollectionInfoV1 extends AggregationInfoV1 { - - private CollectionMode collectionMode; - - public CollectionMode getCollectionMode() { - return collectionMode; - } - - public void setCollectionMode(final CollectionMode collectionMode) { - this.collectionMode = collectionMode; - } - -} diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java deleted file mode 100644 index a9edb7a1..00000000 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java +++ /dev/null @@ -1,21 +0,0 @@ -package eu.dnetlib.openaire.exporter.model.dsm; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; - -/** - * Created by claudio on 29/11/2016. - */ -@JsonAutoDetect -public class CollectionInfoV2 extends AggregationInfo { - - private CollectionMode collectionMode; - - public CollectionMode getCollectionMode() { - return collectionMode; - } - - public void setCollectionMode(final CollectionMode collectionMode) { - this.collectionMode = collectionMode; - } - -} diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java deleted file mode 100644 index 4990daec..00000000 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java +++ /dev/null @@ -1,12 +0,0 @@ -package eu.dnetlib.openaire.exporter.model.dsm; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; - -/** - * Created by claudio on 29/11/2016. - */ -@JsonAutoDetect -@Deprecated -public class TransformationInfoV1 extends AggregationInfoV1 { - -} diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java deleted file mode 100644 index df74986c..00000000 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java +++ /dev/null @@ -1,11 +0,0 @@ -package eu.dnetlib.openaire.exporter.model.dsm; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; - -/** - * Created by claudio on 29/11/2016. - */ -@JsonAutoDetect -public class TransformationInfoV2 extends AggregationInfo { - -} diff --git a/libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java b/libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java new file mode 100644 index 00000000..182f0dd0 --- /dev/null +++ b/libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java @@ -0,0 +1,25 @@ +package eu.dnetlib.openaire.exporter.model.dsm; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class DatasourceApiClientTest { + + @Test + public void testaAgregationHistoryV2_local() throws Exception { + + final ObjectMapper mapper = new ObjectMapper(); + + final AggregationHistoryResponseV2 res = + mapper.readValue(IOUtils.toString(getClass().getResourceAsStream("aggregation-info-v2.json"), "UTF-8"), AggregationHistoryResponseV2.class); + + System.out.println(mapper.writeValueAsString(res)); + + assertTrue(res.getAggregationInfo().size() > 0); + } + +} diff --git a/libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json b/libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json new file mode 100644 index 00000000..d14ca33b --- /dev/null +++ b/libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json @@ -0,0 +1,27 @@ +{ + "header": { + "total": 100, + "page": 0, + "size": 100, + "time": 0, + "statusCode": 0, + "errors": [] + }, + "aggregationInfo": [ + { + "numberOfRecords": 739, + "date": "2024-03-14", + "aggregationStage": "TRANSFORM", + "indexedVersion": false, + "completedSuccessfully": true + }, + { + "numberOfRecords": 822, + "date": "2024-03-14", + "aggregationStage": "COLLECT", + "indexedVersion": false, + "completedSuccessfully": true, + "collectionMode": "REFRESH" + } + ] +}