new dsm api for first collection date and registration date

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java

@@ -28,6 +28,7 @@ import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
 import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
 import eu.dnetlib.openaire.common.AbstractExporterController;
 import eu.dnetlib.openaire.common.OperationManager;
+import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
 import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
 import eu.dnetlib.openaire.dsm.domain.ApiDetails;
 import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
@@ -36,10 +37,12 @@ import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
 import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
 import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis;
 import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
+import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
 import eu.dnetlib.openaire.dsm.domain.RequestFilter;
 import eu.dnetlib.openaire.dsm.domain.RequestSort;
 import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
 import eu.dnetlib.openaire.dsm.domain.Response;
+import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
 import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
 import eu.dnetlib.openaire.vocabularies.Country;
 import io.swagger.v3.oas.annotations.Operation;
@@ -163,9 +166,9 @@ public class DsmApiController extends AbstractExporterController {
 		@ApiResponse(responseCode = "200", description = "OK"),
 		@ApiResponse(responseCode = "500", description = "unexpected error")
 	})
-	public SimpleResponse<?> recentRegistered(@PathVariable final int size) throws Throwable {
+	public SimpleResponse<RegisteredDatasourceInfo> recentRegistered(@PathVariable final int size) throws Throwable {
 		final StopWatch stop = StopWatch.createStarted();
-		final SimpleResponse<?> rsp = dsmCore.searchRecentRegistered(size);
+		final SimpleResponse<RegisteredDatasourceInfo> rsp = dsmCore.searchRecentRegistered(size);
 		return prepareResponse(1, size, stop, rsp);
 	}
 
@@ -422,4 +425,60 @@ public class DsmApiController extends AbstractExporterController {
 			.setSize(size);
 		return rsp;
 	}
+
+	// ------------------------------
+
+	@RequestMapping(value = "/ds/recentregistered/v2/{size}", produces = {
+		"application/json"
+	}, method = RequestMethod.GET)
+	@Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = {
+		DS,
+		R
+	})
+	@ApiResponses(value = {
+		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "500", description = "unexpected error")
+	})
+	public SimpleResponse<SimpleDatasourceInfo> recentRegisteredV2(@PathVariable final int size) throws Throwable {
+		final StopWatch stop = StopWatch.createStarted();
+		final SimpleResponse<SimpleDatasourceInfo> rsp = dsmCore.searchRecentRegisteredV2(size);
+		return prepareResponse(1, size, stop, rsp);
+	}
+
+	@RequestMapping(value = "/ds/countfirstcollect", produces = {
+		"application/json"
+	}, method = RequestMethod.GET)
+	@Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = {
+		DS,
+		R
+	})
+	@ApiResponses(value = {
+		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "500", description = "unexpected error")
+	})
+	public Long countFirstCollectAfter(@RequestParam final String fromDate,
+		@RequestParam(required = false) final String typologyFilter) throws Throwable {
+		return dsmCore.countFirstCollect(fromDate, typologyFilter);
+	}
+
+	@RequestMapping(value = "/ds/firstCollected", produces = {
+		"application/json"
+	}, method = RequestMethod.GET)
+	@Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = {
+		DS,
+		R
+	})
+	@ApiResponses(value = {
+		@ApiResponse(responseCode = "200", description = "OK"),
+		@ApiResponse(responseCode = "500", description = "unexpected error")
+	})
+	public SimpleResponse<SimpleDatasourceInfo> firstCollectedAfter(@RequestParam final String fromDate,
+		@RequestParam(required = false) final String typologyFilter) throws Throwable {
+		final StopWatch stop = StopWatch.createStarted();
+		final List<SimpleDatasourceInfo> list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter);
+		final SimpleResponse<SimpleDatasourceInfo> rsp = ResponseUtils.simpleResponse(list);
+
+		return prepareResponse(1, list.size(), stop, rsp);
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java

@@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.data.domain.Page;
 import org.springframework.jdbc.core.BeanPropertyRowMapper;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowMapper;
 import org.springframework.stereotype.Component;
 
 import eu.dnetlib.enabling.datasources.common.AggregationInfo;
@@ -51,6 +52,7 @@ import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
 import eu.dnetlib.openaire.dsm.domain.RequestFilter;
 import eu.dnetlib.openaire.dsm.domain.RequestSort;
 import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
+import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
 import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
 import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
 import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
@@ -275,7 +277,7 @@ public class DsmCore {
 
 	// HELPERS //////////////
 
-	public SimpleResponse<?> searchRecentRegistered(final int size) throws Throwable {
+	public SimpleResponse<RegisteredDatasourceInfo> searchRecentRegistered(final int size) throws Throwable {
 		try {
 			final String sql =
 				IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset());
@@ -318,4 +320,123 @@ public class DsmCore {
 		return rsp;
 
 	}
+
+	public SimpleResponse<SimpleDatasourceInfo> searchRecentRegisteredV2(final int size) throws Throwable {
+		try {
+			final String sql =
+				IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st"), Charset.defaultCharset());
+
+			final List<SimpleDatasourceInfo> list =
+				jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), size);
+
+			return ResponseUtils.simpleResponse(list);
+		} catch (final Throwable e) {
+			log.error("error searching recent datasources", e);
+			throw e;
+		}
+	}
+
+	public Long countFirstCollect(final String fromDate, final String typeFilter) throws Throwable {
+		try {
+			if (StringUtils.isNotBlank(typeFilter)) {
+				final String sql =
+					IOUtils
+						.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql"), Charset
+							.defaultCharset());
+
+				return jdbcTemplate.queryForObject(sql, Long.class, typeFilter + "%", fromDate);
+			} else {
+				final String sql =
+					IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql"), Charset
+						.defaultCharset());
+
+				return jdbcTemplate.queryForObject(sql, Long.class, fromDate);
+			}
+
+		} catch (final Throwable e) {
+			log.error("error searching datasources using the first collection date", e);
+			throw e;
+		}
+	}
+
+	public List<SimpleDatasourceInfo> getFirstCollectedAfter(final String fromDate, final String typeFilter) throws Throwable {
+		try {
+			if (StringUtils.isNotBlank(typeFilter)) {
+				final String sql =
+					IOUtils
+						.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql"), Charset
+							.defaultCharset());
+
+				return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate);
+
+			} else {
+				final String sql =
+					IOUtils
+						.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql"), Charset
+							.defaultCharset());
+
+				return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), fromDate);
+
+			}
+		} catch (final Throwable e) {
+			log.error("error searching datasources using the first collection date", e);
+			throw e;
+		}
+	}
+
+	private RowMapper<SimpleDatasourceInfo> rowMapperForSimpleDatasourceInfo() {
+
+		return (rs, rowNum) -> {
+			final SimpleDatasourceInfo info = new SimpleDatasourceInfo();
+
+			info.setId(rs.getString("id"));
+			info.setOfficialName(rs.getString("officialName"));
+			info.setEnglishName(rs.getString("englishName"));
+			info.setTypology(rs.getString("typology"));
+			info.setEoscType(rs.getString("eoscType"));
+			info.setEoscDatasourceType(rs.getString("eoscDatasourceType"));
+			info.setRegisteredBy(rs.getString("registeredBy"));
+			info.setRegistrationDate(rs.getString("registrationDate"));
+			info.setFirstCollectionDate(rs.getString("firstCollectionDate"));
+			info.setLastCollectionDate(rs.getString("lastCollectionDate"));
+			info.setLastCollectionTotal(rs.getLong("lastCollectionTotal"));
+
+			final Set<String> compatibilities = new HashSet<>();
+			for (final String s : (String[]) rs.getArray("compatibilities").getArray()) {
+				compatibilities.add(s);
+			}
+
+			// The order of the condition is important
+			if (compatibilities.contains("openaire-cris_1.1")) {
+				info.setCompatibility("openaire-cris_1.1");
+			} else if (compatibilities.contains("openaire4.0")) {
+				info.setCompatibility("openaire4.0");
+			} else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) {
+				info.setCompatibility("driver-openaire2.0");
+			} else if (compatibilities.contains("driver")) {
+				info.setCompatibility("driver");
+			} else if (compatibilities.contains("openaire2.0")) {
+				info.setCompatibility("openaire2.0");
+			} else if (compatibilities.contains("openaire3.0")) {
+				info.setCompatibility("openaire3.0");
+			} else if (compatibilities.contains("openaire2.0_data")) {
+				info.setCompatibility("openaire2.0_data");
+			} else if (compatibilities.contains("native")) {
+				info.setCompatibility("native");
+			} else if (compatibilities.contains("hostedBy")) {
+				info.setCompatibility("hostedBy");
+			} else if (compatibilities.contains("notCompatible")) {
+				info.setCompatibility("notCompatible");
+			} else {
+				info.setCompatibility("UNKNOWN");
+			}
+
+			for (final String s : (String[]) rs.getArray("organizations").getArray()) {
+				info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim());
+			}
+
+			return info;
+		};
+	}
+
 }

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java

@@ -52,8 +52,8 @@ public class ResponseUtils {
 		return header(Lists.newLinkedList(), total);
 	}
 
-	public static SimpleResponse<?> simpleResponse(final List<?> list) {
+	public static <T> SimpleResponse<T> simpleResponse(final List<T> list) {
-		final SimpleResponse rsp = new SimpleResponse().setResponse(list);;
+		final SimpleResponse<T> rsp = new SimpleResponse<T>().setResponse(list);;
 		rsp.setHeader(header(Lists.newLinkedList(), list.size()));
 		return rsp;
 	}

apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java

@@ -0,0 +1,129 @@
+package eu.dnetlib.openaire.dsm.domain;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class SimpleDatasourceInfo {
+
+	private String id;
+	private String officialName;
+	private String englishName;
+	private Map<String, String> organizations = new LinkedHashMap<>();
+	@Deprecated
+	private String typology;
+	private String eoscType;
+	private String eoscDatasourceType;
+	private String registeredBy;
+	private String registrationDate;
+	private String compatibility;
+	private String firstCollectionDate;
+	private String lastCollectionDate;
+	private long lastCollectionTotal;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getOfficialName() {
+		return officialName;
+	}
+
+	public void setOfficialName(final String officialName) {
+		this.officialName = officialName;
+	}
+
+	public String getEnglishName() {
+		return englishName;
+	}
+
+	public void setEnglishName(final String englishName) {
+		this.englishName = englishName;
+	}
+
+	public Map<String, String> getOrganizations() {
+		return organizations;
+	}
+
+	public void setOrganizations(final Map<String, String> organizations) {
+		this.organizations = organizations;
+	}
+
+	@Deprecated
+	public String getTypology() {
+		return typology;
+	}
+
+	@Deprecated
+	public void setTypology(final String typology) {
+		this.typology = typology;
+	}
+
+	public String getEoscType() {
+		return eoscType;
+	}
+
+	public void setEoscType(final String eoscType) {
+		this.eoscType = eoscType;
+	}
+
+	public String getEoscDatasourceType() {
+		return eoscDatasourceType;
+	}
+
+	public void setEoscDatasourceType(final String eoscDatasourceType) {
+		this.eoscDatasourceType = eoscDatasourceType;
+	}
+
+	public String getRegisteredBy() {
+		return registeredBy;
+	}
+
+	public void setRegisteredBy(final String registeredBy) {
+		this.registeredBy = registeredBy;
+	}
+
+	public String getRegistrationDate() {
+		return registrationDate;
+	}
+
+	public void setRegistrationDate(final String registrationDate) {
+		this.registrationDate = registrationDate;
+	}
+
+	public String getCompatibility() {
+		return compatibility;
+	}
+
+	public void setCompatibility(final String compatibility) {
+		this.compatibility = compatibility;
+	}
+
+	public String getFirstCollectionDate() {
+		return firstCollectionDate;
+	}
+
+	public void setFirstCollectionDate(final String firstCollectionDate) {
+		this.firstCollectionDate = firstCollectionDate;
+	}
+
+	public String getLastCollectionDate() {
+		return lastCollectionDate;
+	}
+
+	public void setLastCollectionDate(final String lastCollectionDate) {
+		this.lastCollectionDate = lastCollectionDate;
+	}
+
+	public long getLastCollectionTotal() {
+		return lastCollectionTotal;
+	}
+
+	public void setLastCollectionTotal(final long lastCollectionTotal) {
+		this.lastCollectionTotal = lastCollectionTotal;
+	}
+
+}

apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql

@@ -0,0 +1,7 @@
+select count(*) as count
+from (
+	select d.id
+	from dsm_services d left outer join dsm_api a on (d.id = a.service) 
+	group by d.id
+	having min(a.first_collection_date) >= cast(? as date)
+) as t

apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql

@@ -0,0 +1,8 @@
+select count(*) as count
+from (
+	select d.id
+	from dsm_services d left outer join dsm_api a on (d.id = a.service) 
+	where d._typology_to_remove_ like ?
+	group by d.id
+	having min(a.first_collection_date) >= cast(? as date)
+) as t

apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql

@@ -0,0 +1,29 @@
+SELECT 
+	s.id                                                                    AS "id",
+	s.officialname                                                          AS "officialName",
+	s.englishname                                                           AS "englishName",
+	s._typology_to_remove_                                                  AS "typology",
+	s.eosc_type                                                             AS "eoscType",
+	s.eosc_datasource_type                                                  AS "eoscDatasourceType", 
+	s.registeredby                                                          AS "registeredBy", 
+	s.registrationdate::text                                                AS "registrationDate", 
+	MIN(a.first_collection_date)	                                        AS "firstCollectionDate",
+	MAX(a.last_collection_date)                                             AS "lastCollectionDate", 
+	(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
+	array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL)      AS "compatibilities",
+	array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL)                               AS "organizations"
+FROM 
+	dsm_services s
+	left outer join dsm_api a on (s.id = a.service) 
+	left outer join dsm_service_organization dso on (s.id = dso.service) 
+	left outer join dsm_organizations o on (o.id = dso.organization) 
+GROUP BY 
+	s.id,
+	s.officialname,
+	s.englishname,
+	s._typology_to_remove_,
+	s.eosc_type,
+	s.eosc_datasource_type, 
+	s.registeredby, 
+	s.registrationdate
+HAVING MIN(a.first_collection_date) >= cast(? as date)

apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql

@@ -0,0 +1,31 @@
+SELECT 
+	s.id                                                                    AS "id",
+	s.officialname                                                          AS "officialName",
+	s.englishname                                                           AS "englishName",
+	s._typology_to_remove_                                                  AS "typology",
+	s.eosc_type                                                             AS "eoscType",
+	s.eosc_datasource_type                                                  AS "eoscDatasourceType", 
+	s.registeredby                                                          AS "registeredBy", 
+	s.registrationdate::text                                                AS "registrationDate", 
+	MIN(a.first_collection_date)	                                        AS "firstCollectionDate",
+	MAX(a.last_collection_date)                                             AS "lastCollectionDate", 
+	(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
+	array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL)      AS "compatibilities",
+	array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL)                               AS "organizations"
+FROM 
+	dsm_services s
+	left outer join dsm_api a on (s.id = a.service) 
+	left outer join dsm_service_organization dso on (s.id = dso.service) 
+	left outer join dsm_organizations o on (o.id = dso.organization) 
+WHERE 
+	s._typology_to_remove_ like ?
+GROUP BY 
+	s.id,
+	s.officialname,
+	s.englishname,
+	s._typology_to_remove_,
+	s.eosc_type,
+	s.eosc_datasource_type, 
+	s.registeredby, 
+	s.registrationdate
+HAVING MIN(a.first_collection_date) >= cast(? as date)

apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st

@@ -0,0 +1,38 @@
+SELECT 
+	s.id                                                                    AS "id",
+	s.officialname                                                          AS "officialName",
+	s.englishname                                                           AS "englishName",
+	s._typology_to_remove_                                                  AS "typology",
+	s.eosc_type                                                             AS "eoscType",
+	s.eosc_datasource_type                                                  AS "eoscDatasourceType", 
+	s.registeredby                                                          AS "registeredBy", 
+	s.registrationdate::text                                                AS "registrationDate", 
+	MIN(a.first_collection_date)	                                        AS "firstCollectionDate",
+	MAX(a.last_collection_date)                                             AS "lastCollectionDate", 
+	(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
+	array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL)      AS "compatibilities",
+	array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL)                               AS "organizations"
+FROM 
+	dsm_services s
+	left outer join dsm_api a on (s.id = a.service) 
+	left outer join dsm_service_organization dso on (s.id = dso.service) 
+	left outer join dsm_organizations o on (o.id = dso.organization) 
+WHERE 
+	s.registrationdate is not null 
+	and s.registeredby is not null
+	and s.managed = true
+GROUP BY 
+	s.id,
+	s.officialname,
+	s.englishname,
+	s._typology_to_remove_,
+	s.eosc_type,
+	s.eosc_datasource_type, 
+	s.registeredby, 
+	s.registrationdate
+HAVING
+	s.registrationdate < max(a.last_collection_date)
+	and sum(a.last_collection_total) > 0
+ORDER BY s.registrationdate desc 
+LIMIT ?
+