aggregation dates

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java

@@ -1,14 +1,25 @@
 package eu.dnetlib.openaire.funders;
 
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.stereotype.Component;
 
+import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
+import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
 import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
+import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
+import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
+import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
 import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
 
 @Component
@@ -18,14 +29,24 @@ public class FunderDao {
 	@Autowired
 	private FunderRepository funderRepository;
 
+	@Autowired
+	private MongoLoggerClient mongoLoggerClient;
+
+	private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
+
+	private static final Log log = LogFactory.getLog(FunderDao.class);
+
 	public FunderDbEntry findFunder(final String funderId) throws FundersApiException {
-		return funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID:  " + funderId));
+		return funderRepository.findById(funderId)
+				.map(this::addAggregationHistory)
+				.orElseThrow(() -> new FundersApiException("Funder not found. ID:  " + funderId));
 	}
 
 	public List<FunderDbEntry> listFunders(final int page, final int size) throws FundersApiException {
 		return funderRepository.findAll(PageRequest.of(page, size))
 				.getContent()
 				.stream()
+				.map(this::addAggregationHistory)
 				.collect(Collectors.toList());
 	}
 
@@ -37,4 +58,36 @@ public class FunderDao {
 				.collect(Collectors.toList());
 	}
 
+	public long count() {
+		return funderRepository.count();
+	}
+
+	private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) {
+
+		final List<LocalDate> dates = funder.getDatasources()
+				.stream()
+				.map(FunderDatasource::getId)
+				.map(id -> {
+					try {
+						return mongoLoggerClient.getAggregationHistoryV2(id);
+					} catch (final DsmApiException e) {
+						log.error("Error retrieving the aggregation history");
+						// throw new RuntimeException(e);
+						return new ArrayList<AggregationInfo>();
+					}
+				})
+				.flatMap(List::stream)
+				.filter(AggregationInfo::isCompletedSuccessfully)
+				.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
+				.map(AggregationInfo::getDate)
+				.distinct()
+				.map(s -> LocalDate.parse(s, DATEFORMATTER))
+				.sorted(Comparator.reverseOrder())
+				.toList();
+
+		funder.setAggregationDates(dates);
+
+		return funder;
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java

@@ -32,7 +32,7 @@ public class FundersApiController extends AbstractExporterController {
 	@RequestMapping(value = "/funders/{page}/{size}", produces = {
 			"application/json"
 	}, method = RequestMethod.GET)
-	@Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, registration date")
+	@Operation(summary = "get a page of funders", description = "get a page of funders")
 	@ApiResponses(value = {
 			@ApiResponse(responseCode = "200", description = "OK"),
 			@ApiResponse(responseCode = "500", description = "unexpected error")
@@ -71,4 +71,16 @@ public class FundersApiController extends AbstractExporterController {
 		return fDao.listFunderIds(page, size);
 	}
 
+	@RequestMapping(value = "/funders/count", produces = {
+			"application/json"
+	}, method = RequestMethod.GET)
+	@Operation(summary = "count the funders", description = "count the funders")
+	@ApiResponses(value = {
+			@ApiResponse(responseCode = "200", description = "OK"),
+			@ApiResponse(responseCode = "500", description = "unexpected error")
+	})
+	public long countFunders() throws FundersApiException {
+		return fDao.count();
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.openaire.funders.domain.db;
+
+import java.io.Serializable;
+
+public class FunderDatasource implements Serializable {
+
+	private static final long serialVersionUID = 2145493560459874509L;
+
+	private String id;
+
+	private String name;
+
+	private String type;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java

@@ -58,6 +58,10 @@ public class FunderDbEntry implements Serializable {
 	@Column(name = "pids", columnDefinition = "jsonb")
 	private List<FunderPid> pids;
 
+	@Type(type = "json")
+	@Column(name = "datasources", columnDefinition = "jsonb")
+	private List<FunderDatasource> datasources;
+
 	@Transient
 	private List<LocalDate> aggregationDates;
 
@@ -140,4 +144,12 @@ public class FunderDbEntry implements Serializable {
 	public void setPids(final List<FunderPid> pids) {
 		this.pids = pids;
 	}
+
+	public List<FunderDatasource> getDatasources() {
+		return datasources;
+	}
+
+	public void setDatasources(final List<FunderDatasource> datasources) {
+		this.datasources = datasources;
+	}
 }

apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql

@@ -9,6 +9,7 @@ CREATE VIEW funders_view AS SELECT
 	o.country           AS country,
 	o.dateofcollection  AS registrationdate,
 	o.registered_funder AS registered,
+	CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources,
 	CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids
 FROM 
 	dsm_organizations o