D-Net
/
dnet-applications
23
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

merged the sub classes in the main class

This commit is contained in:
Michele Artini 2024-03-18 11:19:16 +01:00
parent d2da1a1270
commit ed73d524d8
10 changed files with 209 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java
View File

@ -48,11 +48,7 @@ import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfoV1;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV1;
import eu.dnetlib.openaire.exporter.model.dsm.CollectionInfoV2;
import eu.dnetlib.openaire.exporter.model.dsm.CollectionMode;
import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV1;
import eu.dnetlib.openaire.exporter.model.dsm.TransformationInfoV2;
import eu.dnetlib.openaire.info.JdbcInfoDao;
/**
@ -75,16 +71,16 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
private final static String LOADTIME = "loadtime";
private final LoadingCache<String, Instant> loadingCache =
CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
CacheBuilder.newBuilder().maximumSize(1).expireAfterWrite(60, TimeUnit.MINUTES).build(new CacheLoader<String, Instant>() {
// The only cached value is associated to "loadtime"
@Override
public Instant load(final String key) {
final Instant loadTime = getLoadTime();
log.debug("found load time: " + loadTime.toString());
return loadTime;
}
});
// The only cached value is associated to "loadtime"
@Override
public Instant load(final String key) {
final Instant loadTime = getLoadTime();
log.debug("found load time: " + loadTime.toString());
return loadTime;
}
});
private static final Bson fields = getFields();
@ -94,30 +90,64 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
@Cacheable("dsm-aggregationhistory-cache-v1")
@Deprecated
public List<AggregationInfoV1> getAggregationHistoryV1(final String dsId) throws DsmApiException {
return getAggregationHistory(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
return getAggregationHistoryV1(dsId, queryForAggregationHistoryV1(dsId, "(collect|transform)"), getMapperV1());
}
@Override
@Cacheable("dsm-aggregationhistory-cache-v2")
public List<AggregationInfo> getAggregationHistoryV2(final String dsId) throws DsmApiException {
return getAggregationHistory(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
return getAggregationHistoryV2(dsId, queryForAggregationHistoryV2(dsId, "(collect|transform)"), getMapperV2());
}
private <T extends AggregationInfo> List<T> getAggregationHistory(final String dsId,
final Bson queryForAggregationHistory,
final Function<Document, T> mapper) throws DsmApiException {
@Deprecated
private List<AggregationInfoV1> getAggregationHistoryV1(final String dsId,
final Bson queryForAggregationHistory,
final Function<Document, AggregationInfoV1> mapper) throws DsmApiException {
log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
final Datasource conf = config.getDatasource();
try {
final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
.projection(fields)
.limit(conf.getMongoQueryLimit())
.sort(dbo("system:startHumanDate", -1));
.projection(fields)
.limit(conf.getMongoQueryLimit())
.sort(dbo("system:startHumanDate", -1));
final List<T> aggregationInfos = Utils.stream(aggregationDocs.iterator())
.map(mapper)
.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
.collect(Collectors.toList());
final List<AggregationInfoV1> aggregationInfos = Utils.stream(aggregationDocs.iterator())
.map(mapper)
.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
.collect(Collectors.toList());
final Instant loadTime = loadingCache.get(LOADTIME);
if (!Objects.equals(Instant.MIN, loadTime)) {
for (final AggregationInfoV1 a : aggregationInfos) {
if (asInstant(a).isBefore(loadTime) && AggregationStage.COLLECT.equals(a.getAggregationStage())) {
a.setIndexedVersion(true);
break;
}
}
}
return aggregationInfos;
} catch (final Throwable e) {
throw new DsmApiException(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.format("error reading aggregation history for '%s'", dsId), e);
}
}
private List<AggregationInfo> getAggregationHistoryV2(final String dsId,
final Bson queryForAggregationHistory,
final Function<Document, AggregationInfo> mapper) throws DsmApiException {
log.warn(String.format("getAggregationHistory(dsId = %s): not using cache", dsId));
final Datasource conf = config.getDatasource();
try {
final FindIterable<Document> aggregationDocs = getCollection().find(queryForAggregationHistory)
.projection(fields)
.limit(conf.getMongoQueryLimit())
.sort(dbo("system:startHumanDate", -1));
final List<AggregationInfo> aggregationInfos = Utils.stream(aggregationDocs.iterator())
.map(mapper)
.filter(ai -> ai.getNumberOfRecords() >= 0 && StringUtils.isNotBlank(ai.getDate()))
.collect(Collectors.toList());
final Instant loadTime = loadingCache.get(LOADTIME);
@ -146,9 +176,14 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
return Instant.parse(a.getDate() + "T00:00:00Z");
}
@Deprecated
private Instant asInstant(final AggregationInfoV1 a) {
return Instant.parse(a.getDate() + "T00:00:00Z");
}
@Override
@CacheEvict(cacheNames = {
"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
"dsm-aggregationhistory-cache-v1", "dsm-aggregationhistory-cache-v2", "dsm-firstharvestdate-cache"
}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
@ -166,20 +201,18 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
switch (stage) {
case COLLECT:
final CollectionInfoV1 cInfo = new CollectionInfoV1();
final AggregationInfoV1 cInfo = new AggregationInfoV1();
cInfo.setAggregationStage(stage);
cInfo.setCollectionMode(getCollectionMode(d));
cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
cInfo.setDate(getDate(d));
cInfo.setCompletedSuccessfully(success);
info = cInfo;
break;
case TRANSFORM:
final TransformationInfoV1 tInfo = new TransformationInfoV1();
final AggregationInfoV1 tInfo = new AggregationInfoV1();
tInfo.setAggregationStage(stage);
tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
tInfo.setDate(getDate(d));
tInfo.setCompletedSuccessfully(success);
info = tInfo;
break;
}
@ -197,7 +230,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
switch (stage) {
case COLLECT:
final CollectionInfoV2 cInfo = new CollectionInfoV2();
final AggregationInfo cInfo = new AggregationInfo();
cInfo.setAggregationStage(stage);
cInfo.setCollectionMode(getCollectionMode(d));
cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
@ -206,7 +239,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
info = cInfo;
break;
case TRANSFORM:
final TransformationInfoV2 tInfo = new TransformationInfoV2();
final AggregationInfo tInfo = new AggregationInfo();
tInfo.setAggregationStage(stage);
tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
tInfo.setDate(getDate(d));
@ -220,23 +253,19 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
private CollectionMode getCollectionMode(final Document d) {
return Optional.ofNullable(d.getString("system:node:SELECT_MODE:selection"))
.map(CollectionMode::valueOf)
.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
.map(CollectionMode::valueOf)
.orElse(null));
.orElseGet(() -> Optional.ofNullable(d.getString("collectionMode"))
.map(CollectionMode::valueOf)
.orElse(null));
}
private Integer getNumberOfRecords(final Document d) {
final String sinkSize = d.getString("mainlog:sinkSize");
final String total = d.getString("mainlog:total");
if (StringUtils.isNotBlank(sinkSize)) {
return Ints.tryParse(sinkSize);
} else if (StringUtils.isNotBlank(total)) {
return Ints.tryParse(total);
} else {
return -1;
}
if (StringUtils.isNotBlank(sinkSize)) { return Ints.tryParse(sinkSize); }
if (StringUtils.isNotBlank(total)) { return Ints.tryParse(total); }
return -1;
}
private String getDate(final Document d) {

4
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationHistoryResponseV2.java
View File

@ -9,6 +9,10 @@ public class AggregationHistoryResponseV2 extends Response {
private List<AggregationInfo> aggregationInfo;
public AggregationHistoryResponseV2() {
super();
}
public AggregationHistoryResponseV2(final List<AggregationInfo> aggregationInfo) {
super();
this.aggregationInfo = aggregationInfo;

23
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfo.java
View File

@ -1,6 +1,16 @@
package eu.dnetlib.openaire.exporter.model.dsm;
public abstract class AggregationInfo {
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@JsonAutoDetect
@JsonInclude(Include.NON_NULL)
public class AggregationInfo implements Serializable {
private static final long serialVersionUID = -4908395195618212510L;
private int numberOfRecords;
@ -12,6 +22,8 @@ public abstract class AggregationInfo {
private boolean completedSuccessfully = true;
private CollectionMode collectionMode;
public AggregationInfo() {}
public int getNumberOfRecords() {
@ -53,4 +65,13 @@ public abstract class AggregationInfo {
public void setCompletedSuccessfully(final boolean completedSuccessfully) {
this.completedSuccessfully = completedSuccessfully;
}
public CollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(final CollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
}

66
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/AggregationInfoV1.java
View File

@ -1,13 +1,67 @@
package eu.dnetlib.openaire.exporter.model.dsm;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.io.Serializable;
public class AggregationInfoV1 extends AggregationInfo {
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@Override
@JsonIgnore
public boolean isCompletedSuccessfully() {
return super.isCompletedSuccessfully();
@JsonAutoDetect
@JsonInclude(Include.NON_NULL)
public class AggregationInfoV1 implements Serializable {
private static final long serialVersionUID = 7333873121568523946L;
private int numberOfRecords;
private String date;
private AggregationStage aggregationStage;
private boolean indexedVersion = false;
private CollectionMode collectionMode;
public AggregationInfoV1() {}
public int getNumberOfRecords() {
return numberOfRecords;
}
public void setNumberOfRecords(final int numberOfRecords) {
this.numberOfRecords = numberOfRecords;
}
public String getDate() {
return date;
}
public void setDate(final String date) {
this.date = date;
}
public AggregationStage getAggregationStage() {
return aggregationStage;
}
public void setAggregationStage(final AggregationStage aggregationStage) {
this.aggregationStage = aggregationStage;
}
public boolean isIndexedVersion() {
return indexedVersion;
}
public void setIndexedVersion(final boolean indexedVersion) {
this.indexedVersion = indexedVersion;
}
public CollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(final CollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
}

22
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV1.java
View File

@ -1,22 +0,0 @@
package eu.dnetlib.openaire.exporter.model.dsm;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 29/11/2016.
*/
@JsonAutoDetect
@Deprecated
public class CollectionInfoV1 extends AggregationInfoV1 {
private CollectionMode collectionMode;
public CollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(final CollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
}

21
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/CollectionInfoV2.java
View File

@ -1,21 +0,0 @@
package eu.dnetlib.openaire.exporter.model.dsm;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 29/11/2016.
*/
@JsonAutoDetect
public class CollectionInfoV2 extends AggregationInfo {
private CollectionMode collectionMode;
public CollectionMode getCollectionMode() {
return collectionMode;
}
public void setCollectionMode(final CollectionMode collectionMode) {
this.collectionMode = collectionMode;
}
}

12
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV1.java
View File

@ -1,12 +0,0 @@
package eu.dnetlib.openaire.exporter.model.dsm;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 29/11/2016.
*/
@JsonAutoDetect
@Deprecated
public class TransformationInfoV1 extends AggregationInfoV1 {
}

11
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/dsm/TransformationInfoV2.java
View File

@ -1,11 +0,0 @@
package eu.dnetlib.openaire.exporter.model.dsm;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
/**
* Created by claudio on 29/11/2016.
*/
@JsonAutoDetect
public class TransformationInfoV2 extends AggregationInfo {
}

25
libs/dnet-exporter-model/src/test/java/eu/dnetlib/openaire/exporter/model/dsm/DatasourceApiClientTest.java Normal file
View File

@ -0,0 +1,25 @@
package eu.dnetlib.openaire.exporter.model.dsm;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
public class DatasourceApiClientTest {
@Test
public void testaAgregationHistoryV2_local() throws Exception {
final ObjectMapper mapper = new ObjectMapper();
final AggregationHistoryResponseV2 res =
mapper.readValue(IOUtils.toString(getClass().getResourceAsStream("aggregation-info-v2.json"), "UTF-8"), AggregationHistoryResponseV2.class);
System.out.println(mapper.writeValueAsString(res));
assertTrue(res.getAggregationInfo().size() > 0);
}
}

27
libs/dnet-exporter-model/src/test/resources/eu/dnetlib/openaire/exporter/model/dsm/aggregation-info-v2.json Normal file
View File

@ -0,0 +1,27 @@
{
"header": {
"total": 100,
"page": 0,
"size": 100,
"time": 0,
"statusCode": 0,
"errors": []
},
"aggregationInfo": [
{
"numberOfRecords": 739,
"date": "2024-03-14",
"aggregationStage": "TRANSFORM",
"indexedVersion": false,
"completedSuccessfully": true
},
{
"numberOfRecords": 822,
"date": "2024-03-14",
"aggregationStage": "COLLECT",
"indexedVersion": false,
"completedSuccessfully": true,
"collectionMode": "REFRESH"
}
]
}