Merge branch 'master' into old_wf_history_for_beta_and_prod

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java

@@ -28,7 +28,6 @@ import org.springframework.jdbc.core.BeanPropertyRowMapper;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
 import eu.dnetlib.enabling.datasources.common.DsmException;
 import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
 import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
@@ -38,6 +37,7 @@ import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
 import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
 import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
 import eu.dnetlib.openaire.dsm.dao.VocabularyClient;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
 import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
 import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
 import eu.dnetlib.openaire.dsm.domain.ApiDetails;

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClient.java

@@ -1,10 +1,10 @@
 package eu.dnetlib.openaire.dsm.dao;
 
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
-import eu.dnetlib.enabling.datasources.common.DsmException;
-
 import java.util.List;
 
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
+
 public interface MongoLoggerClient {
 
 	List<AggregationInfo> getAggregationHistory(final String dsId) throws DsmException;

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/MongoLoggerClientImpl.java

@@ -15,7 +15,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateFormatUtils;
 import org.apache.commons.logging.Log;
@@ -41,11 +40,11 @@ import com.mongodb.client.MongoCollection;
 
 import eu.dnetlib.DnetOpenaireExporterProperties;
 import eu.dnetlib.DnetOpenaireExporterProperties.Datasource;
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
 import eu.dnetlib.enabling.datasources.common.AggregationStage;
 import eu.dnetlib.enabling.datasources.common.DsmException;
 import eu.dnetlib.miscutils.datetime.DateUtils;
 import eu.dnetlib.openaire.common.Utils;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
 import eu.dnetlib.openaire.dsm.dao.utils.DsmMappingUtils;
 import eu.dnetlib.openaire.dsm.domain.CollectionInfo;
 import eu.dnetlib.openaire.dsm.domain.CollectionMode;
@@ -149,7 +148,6 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 
 				AggregationInfo info = null;
 				final AggregationStage stage = AggregationStage.parse(d.getString("system:wfName"));
-				final boolean success = isCompletedSuccesfully(d);
 
 				switch (stage) {
 
@@ -157,17 +155,15 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 					final CollectionInfo cInfo = new CollectionInfo();
 					cInfo.setAggregationStage(stage);
 					cInfo.setCollectionMode(getCollectionMode(d));
-					cInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
+					cInfo.setNumberOfRecords(getNumberOfRecords(d));
 					cInfo.setDate(getDate(d));
-					cInfo.setCompletedSuccessfully(success);
 					info = cInfo;
 					break;
 				case TRANSFORM:
 					final TransformationInfo tInfo = new TransformationInfo();
 					tInfo.setAggregationStage(stage);
-					tInfo.setNumberOfRecords(success ? getNumberOfRecords(d) : 0);
+					tInfo.setNumberOfRecords(getNumberOfRecords(d));
 					tInfo.setDate(getDate(d));
-					tInfo.setCompletedSuccessfully(success);
 					info = tInfo;
 					break;
 				}
@@ -203,13 +199,8 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 		return DateFormatUtils.format(new DateUtils().parse(dateString), DsmMappingUtils.DATE_FORMAT);
 	}
 
-	private boolean isCompletedSuccesfully(final Document d) {
-		final String boolString = d.getString("system:isCompletedSuccessfully");
-		return BooleanUtils.toBoolean(boolString);
-	}
-
 	private static Bson getFields() {
-		return fields(eq("system:wfName", 1), eq("system:node:SELECT_MODE:selection", 1), eq("collectionMode", 1), eq("mainlog:sinkSize", 1), eq("mainlog:writeOps", 1), eq("mainlog:total", 1), eq("system:startHumanDate", 1), eq("system:profileName", 1), eq("system:isCompletedSuccessfully", 1));
+		return fields(eq("system:wfName", 1), eq("system:node:SELECT_MODE:selection", 1), eq("collectionMode", 1), eq("mainlog:sinkSize", 1), eq("mainlog:writeOps", 1), eq("mainlog:total", 1), eq("system:startHumanDate", 1), eq("system:profileName", 1));
 	}
 
 	private static BasicDBObject dbo(final String key, final Object value) {
@@ -217,7 +208,7 @@ public class MongoLoggerClientImpl implements MongoLoggerClient {
 	}
 
 	private Bson queryForAggregationHistory(final String dsId, final String pattern) {
-		return and(eq("parentDatasourceId", dsId), eq("system:profileFamily", "aggregator"), regex("system:wfName", pattern, "i"));
+		return and(eq("parentDatasourceId", dsId), eq("system:profileFamily", "aggregator"), eq("system:isCompletedSuccessfully", "true"), regex("system:wfName", pattern, "i"));
 	}
 
 	private synchronized MongoCollection<Document> getCollection() {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/patched/AggregationInfo.java

@@ -0,0 +1,49 @@
+package eu.dnetlib.openaire.dsm.dao.patched;
+
+import eu.dnetlib.enabling.datasources.common.AggregationStage;
+
+public abstract class AggregationInfo {
+
+	private int numberOfRecords;
+
+	private String date;
+
+	private AggregationStage aggregationStage;
+
+	private boolean indexedVersion = false;
+
+	public AggregationInfo() {}
+
+	public int getNumberOfRecords() {
+		return numberOfRecords;
+	}
+
+	public void setNumberOfRecords(final int numberOfRecords) {
+		this.numberOfRecords = numberOfRecords;
+	}
+
+	public String getDate() {
+		return date;
+	}
+
+	public void setDate(final String date) {
+		this.date = date;
+	}
+
+	public AggregationStage getAggregationStage() {
+		return aggregationStage;
+	}
+
+	public void setAggregationStage(final AggregationStage aggregationStage) {
+		this.aggregationStage = aggregationStage;
+	}
+
+	public boolean isIndexedVersion() {
+		return indexedVersion;
+	}
+
+	public void setIndexedVersion(final boolean indexedVersion) {
+		this.indexedVersion = indexedVersion;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/AggregationHistoryResponse.java

@@ -4,7 +4,7 @@ import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
 
 @JsonAutoDetect
 public class AggregationHistoryResponse extends Response {

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/CollectionInfo.java

@@ -2,8 +2,7 @@ package eu.dnetlib.openaire.dsm.domain;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
-import io.swagger.v3.oas.annotations.media.Schema;
 
 /**
  * Created by claudio on 29/11/2016.

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/DatasourceInfo.java

@@ -4,7 +4,7 @@ import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
 import io.swagger.v3.oas.annotations.media.Schema;
 
 @JsonAutoDetect

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/TransformationInfo.java

@@ -2,7 +2,7 @@ package eu.dnetlib.openaire.dsm.domain;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 
-import eu.dnetlib.enabling.datasources.common.AggregationInfo;
+import eu.dnetlib.openaire.dsm.dao.patched.AggregationInfo;
 
 /**
  * Created by claudio on 29/11/2016.