D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

create a patched version of AggregationInfo

This commit is contained in:
Michele Artini 2022-09-09 14:47:18 +02:00
parent 8e7901afdd
commit 9985178289
8 changed files with 62 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java
View File

@ -28,7 +28,6 @@ import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
@ -38,6 +37,7 @@ import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
import eu.dnetlib.openaire.dsm.dao.VocabularyClient;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;

6
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java
View File

@ -1,10 +1,10 @@
package eu.dnetlib.openaire.dsm.dao;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.DsmException;
import java.util.List;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
public interface MongoLoggerClient {
List<AggregationInfo> getAggregationHistory(final String dsId) throws DsmException;

19
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java
View File

@ -15,7 +15,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.logging.Log;
@ -41,11 +40,11 @@ import com.mongodb.client.MongoCollection;
import eu.dnetlib.DnetOpenaireExporterProperties;
import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.enabling.datasources.common.AggregationStage;
import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.miscutils.datetime.DateUtils;
import eu.dnetlib.openaire.common.Utils;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
import eu.dnetlib.openaire.dsm.domain.CollectionMode;
@ -149,7 +148,6 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
AggregationInfo info = null;
final AggregationStage stage = AggregationStage.parse(d.getString("system:wfName"));
final boolean success = isCompletedSuccesfully(d);
switch (stage) {
@ -157,17 +155,15 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
final CollectionInfo cInfo = new CollectionInfo();
cInfo.setAggregationStage(stage);
cInfo.setCollectionMode(getCollectionMode(d));
cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
cInfo.setNumberOfRecords(getNumberOfRecords(d));
cInfo.setDate(getDate(d));
cInfo.setCompletedSuccessfully(success);
info = cInfo;
break;
case TRANSFORM:
final TransformationInfo tInfo = new TransformationInfo();
tInfo.setAggregationStage(stage);
tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
tInfo.setNumberOfRecords(getNumberOfRecords(d));
tInfo.setDate(getDate(d));
tInfo.setCompletedSuccessfully(success);
info = tInfo;
break;
}
@ -203,13 +199,8 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
return DateFormatUtils.format(new DateUtils().parse(dateString), DsmMappingUtils.DATE_FORMAT);
}
private boolean isCompletedSuccesfully(final Document d) {
final String boolString = d.getString("system:isCompletedSuccessfully");
return BooleanUtils.toBoolean(boolString);
}
private static Bson getFields() {
return fields(eq("system:wfName", 1), eq("system:node:SELECT_MODE:selection", 1), eq("collectionMode", 1), eq("mainlog:sinkSize", 1), eq("mainlog:writeOps", 1), eq("mainlog:total", 1), eq("system:startHumanDate", 1), eq("system:profileName", 1), eq("system:isCompletedSuccessfully", 1));
return fields(eq("system:wfName", 1), eq("system:node:SELECT_MODE:selection", 1), eq("collectionMode", 1), eq("mainlog:sinkSize", 1), eq("mainlog:writeOps", 1), eq("mainlog:total", 1), eq("system:startHumanDate", 1), eq("system:profileName", 1));
}
private static BasicDBObject dbo(final String key, final Object value) {
@ -217,7 +208,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
}
private Bson queryForAggregationHistory(final String dsId, final String pattern) {
return and(eq("parentDatasourceId", dsId), eq("system:profileFamily", "aggregator"), regex("system:wfName", pattern, "i"));
return and(eq("parentDatasourceId", dsId), eq("system:profileFamily", "aggregator"), eq("system:isCompletedSuccessfully", "true"), regex("system:wfName", pattern, "i"));
}
private synchronized MongoCollection<Document> getCollection() {

49
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/patched/AggregationInfo.java Normal file
View File

@ -0,0 +1,49 @@
package eu.dnetlib.openaire.dsm.dao.patched;
import eu.dnetlib.enabling.datasources.common.AggregationStage;
public abstract class AggregationInfo {
private int numberOfRecords;
private String date;
private AggregationStage aggregationStage;
private boolean indexedVersion = false;
public AggregationInfo() {}
public int getNumberOfRecords() {
return numberOfRecords;
}
public void setNumberOfRecords(final int numberOfRecords) {
this.numberOfRecords = numberOfRecords;
}
public String getDate() {
return date;
}
public void setDate(final String date) {
this.date = date;
}
public AggregationStage getAggregationStage() {
return aggregationStage;
}
public void setAggregationStage(final AggregationStage aggregationStage) {
this.aggregationStage = aggregationStage;
}
public boolean isIndexedVersion() {
return indexedVersion;
}
public void setIndexedVersion(final boolean indexedVersion) {
this.indexedVersion = indexedVersion;
}
}

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java
View File

@ -4,7 +4,7 @@ import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
@JsonAutoDetect
public class AggregationHistoryResponse extends Response {

3
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java
View File

@ -2,8 +2,7 @@ package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import io.swagger.v3.oas.annotations.media.Schema;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
/**
* Created by claudio on 29/11/2016.

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java
View File

@ -4,7 +4,7 @@ import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
import io.swagger.v3.oas.annotations.media.Schema;
@JsonAutoDetect

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java
View File

@ -2,7 +2,7 @@ package eu.dnetlib.openaire.dsm.domain;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
/**
* Created by claudio on 29/11/2016.