diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java index 329923ef..99450dc5 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -28,6 +28,7 @@ import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.OperationManager; +import eu.dnetlib.openaire.dsm.dao.ResponseUtils; import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; import eu.dnetlib.openaire.dsm.domain.ApiDetails; import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; @@ -36,10 +37,12 @@ import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis; import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; import eu.dnetlib.openaire.dsm.domain.Response; +import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.vocabularies.Country; import io.swagger.v3.oas.annotations.Operation; @@ -163,9 +166,9 @@ public class DsmApiController extends AbstractExporterController { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { + public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { final StopWatch stop = StopWatch.createStarted(); - final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); + final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); return prepareResponse(1, size, stop, rsp); } @@ -422,4 +425,60 @@ public class DsmApiController extends AbstractExporterController { .setSize(size); return rsp; } + + // ------------------------------ + + @RequestMapping(value = "/ds/recentregistered/v2/{size}", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public SimpleResponse recentRegisteredV2(@PathVariable final int size) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final SimpleResponse rsp = dsmCore.searchRecentRegisteredV2(size); + return prepareResponse(1, size, stop, rsp); + } + + @RequestMapping(value = "/ds/countfirstcollect", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Long countFirstCollectAfter(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + return dsmCore.countFirstCollect(fromDate, typologyFilter); + } + + @RequestMapping(value = "/ds/firstCollected", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public SimpleResponse firstCollectedAfter(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final List list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter); + final SimpleResponse rsp = ResponseUtils.simpleResponse(list); + + return prepareResponse(1, list.size(), stop, rsp); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java index 13782b8c..42911d5b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.domain.Page; import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Component; import eu.dnetlib.enabling.datasources.common.AggregationInfo; @@ -51,6 +52,7 @@ import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; @@ -275,7 +277,7 @@ public class DsmCore { // HELPERS ////////////// - public SimpleResponse searchRecentRegistered(final int size) throws Throwable { + public SimpleResponse searchRecentRegistered(final int size) throws Throwable { try { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); @@ -318,4 +320,123 @@ public class DsmCore { return rsp; } + + public SimpleResponse searchRecentRegisteredV2(final int size) throws Throwable { + try { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st"), Charset.defaultCharset()); + + final List list = + jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), size); + + return ResponseUtils.simpleResponse(list); + } catch (final Throwable e) { + log.error("error searching recent datasources", e); + throw e; + } + } + + public Long countFirstCollect(final String fromDate, final String typeFilter) throws Throwable { + try { + if (StringUtils.isNotBlank(typeFilter)) { + final String sql = + IOUtils + .toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.queryForObject(sql, Long.class, typeFilter + "%", fromDate); + } else { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.queryForObject(sql, Long.class, fromDate); + } + + } catch (final Throwable e) { + log.error("error searching datasources using the first collection date", e); + throw e; + } + } + + public List getFirstCollectedAfter(final String fromDate, final String typeFilter) throws Throwable { + try { + if (StringUtils.isNotBlank(typeFilter)) { + final String sql = + IOUtils + .toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate); + + } else { + final String sql = + IOUtils + .toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), fromDate); + + } + } catch (final Throwable e) { + log.error("error searching datasources using the first collection date", e); + throw e; + } + } + + private RowMapper rowMapperForSimpleDatasourceInfo() { + + return (rs, rowNum) -> { + final SimpleDatasourceInfo info = new SimpleDatasourceInfo(); + + info.setId(rs.getString("id")); + info.setOfficialName(rs.getString("officialName")); + info.setEnglishName(rs.getString("englishName")); + info.setTypology(rs.getString("typology")); + info.setEoscType(rs.getString("eoscType")); + info.setEoscDatasourceType(rs.getString("eoscDatasourceType")); + info.setRegisteredBy(rs.getString("registeredBy")); + info.setRegistrationDate(rs.getString("registrationDate")); + info.setFirstCollectionDate(rs.getString("firstCollectionDate")); + info.setLastCollectionDate(rs.getString("lastCollectionDate")); + info.setLastCollectionTotal(rs.getLong("lastCollectionTotal")); + + final Set compatibilities = new HashSet<>(); + for (final String s : (String[]) rs.getArray("compatibilities").getArray()) { + compatibilities.add(s); + } + + // The order of the condition is important + if (compatibilities.contains("openaire-cris_1.1")) { + info.setCompatibility("openaire-cris_1.1"); + } else if (compatibilities.contains("openaire4.0")) { + info.setCompatibility("openaire4.0"); + } else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) { + info.setCompatibility("driver-openaire2.0"); + } else if (compatibilities.contains("driver")) { + info.setCompatibility("driver"); + } else if (compatibilities.contains("openaire2.0")) { + info.setCompatibility("openaire2.0"); + } else if (compatibilities.contains("openaire3.0")) { + info.setCompatibility("openaire3.0"); + } else if (compatibilities.contains("openaire2.0_data")) { + info.setCompatibility("openaire2.0_data"); + } else if (compatibilities.contains("native")) { + info.setCompatibility("native"); + } else if (compatibilities.contains("hostedBy")) { + info.setCompatibility("hostedBy"); + } else if (compatibilities.contains("notCompatible")) { + info.setCompatibility("notCompatible"); + } else { + info.setCompatibility("UNKNOWN"); + } + + for (final String s : (String[]) rs.getArray("organizations").getArray()) { + info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim()); + } + + return info; + }; + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java index a2d422b8..cdc4c581 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java @@ -52,8 +52,8 @@ public class ResponseUtils { return header(Lists.newLinkedList(), total); } - public static SimpleResponse simpleResponse(final List list) { - final SimpleResponse rsp = new SimpleResponse().setResponse(list);; + public static SimpleResponse simpleResponse(final List list) { + final SimpleResponse rsp = new SimpleResponse().setResponse(list);; rsp.setHeader(header(Lists.newLinkedList(), list.size())); return rsp; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java new file mode 100644 index 00000000..997f3894 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java @@ -0,0 +1,129 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class SimpleDatasourceInfo { + + private String id; + private String officialName; + private String englishName; + private Map organizations = new LinkedHashMap<>(); + @Deprecated + private String typology; + private String eoscType; + private String eoscDatasourceType; + private String registeredBy; + private String registrationDate; + private String compatibility; + private String firstCollectionDate; + private String lastCollectionDate; + private long lastCollectionTotal; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialName() { + return officialName; + } + + public void setOfficialName(final String officialName) { + this.officialName = officialName; + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(final String englishName) { + this.englishName = englishName; + } + + public Map getOrganizations() { + return organizations; + } + + public void setOrganizations(final Map organizations) { + this.organizations = organizations; + } + + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public void setTypology(final String typology) { + this.typology = typology; + } + + public String getEoscType() { + return eoscType; + } + + public void setEoscType(final String eoscType) { + this.eoscType = eoscType; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + } + + public String getRegisteredBy() { + return registeredBy; + } + + public void setRegisteredBy(final String registeredBy) { + this.registeredBy = registeredBy; + } + + public String getRegistrationDate() { + return registrationDate; + } + + public void setRegistrationDate(final String registrationDate) { + this.registrationDate = registrationDate; + } + + public String getCompatibility() { + return compatibility; + } + + public void setCompatibility(final String compatibility) { + this.compatibility = compatibility; + } + + public String getFirstCollectionDate() { + return firstCollectionDate; + } + + public void setFirstCollectionDate(final String firstCollectionDate) { + this.firstCollectionDate = firstCollectionDate; + } + + public String getLastCollectionDate() { + return lastCollectionDate; + } + + public void setLastCollectionDate(final String lastCollectionDate) { + this.lastCollectionDate = lastCollectionDate; + } + + public long getLastCollectionTotal() { + return lastCollectionTotal; + } + + public void setLastCollectionTotal(final long lastCollectionTotal) { + this.lastCollectionTotal = lastCollectionTotal; + } + +} diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql new file mode 100644 index 00000000..1cf5f393 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql @@ -0,0 +1,7 @@ +select count(*) as count +from ( + select d.id + from dsm_services d left outer join dsm_api a on (d.id = a.service) + group by d.id + having min(a.first_collection_date) >= cast(? as date) +) as t diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql new file mode 100644 index 00000000..b0a79a29 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql @@ -0,0 +1,8 @@ +select count(*) as count +from ( + select d.id + from dsm_services d left outer join dsm_api a on (d.id = a.service) + where d._typology_to_remove_ like ? + group by d.id + having min(a.first_collection_date) >= cast(? as date) +) as t diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql new file mode 100644 index 00000000..f228e03b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql @@ -0,0 +1,29 @@ +SELECT + s.id AS "id", + s.officialname AS "officialName", + s.englishname AS "englishName", + s._typology_to_remove_ AS "typology", + s.eosc_type AS "eoscType", + s.eosc_datasource_type AS "eoscDatasourceType", + s.registeredby AS "registeredBy", + s.registrationdate::text AS "registrationDate", + MIN(a.first_collection_date) AS "firstCollectionDate", + MAX(a.last_collection_date) AS "lastCollectionDate", + (array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal", + array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities", + array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations" +FROM + dsm_services s + left outer join dsm_api a on (s.id = a.service) + left outer join dsm_service_organization dso on (s.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +GROUP BY + s.id, + s.officialname, + s.englishname, + s._typology_to_remove_, + s.eosc_type, + s.eosc_datasource_type, + s.registeredby, + s.registrationdate +HAVING MIN(a.first_collection_date) >= cast(? as date) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql new file mode 100644 index 00000000..13d64cb4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql @@ -0,0 +1,31 @@ +SELECT + s.id AS "id", + s.officialname AS "officialName", + s.englishname AS "englishName", + s._typology_to_remove_ AS "typology", + s.eosc_type AS "eoscType", + s.eosc_datasource_type AS "eoscDatasourceType", + s.registeredby AS "registeredBy", + s.registrationdate::text AS "registrationDate", + MIN(a.first_collection_date) AS "firstCollectionDate", + MAX(a.last_collection_date) AS "lastCollectionDate", + (array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal", + array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities", + array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations" +FROM + dsm_services s + left outer join dsm_api a on (s.id = a.service) + left outer join dsm_service_organization dso on (s.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +WHERE + s._typology_to_remove_ like ? +GROUP BY + s.id, + s.officialname, + s.englishname, + s._typology_to_remove_, + s.eosc_type, + s.eosc_datasource_type, + s.registeredby, + s.registrationdate +HAVING MIN(a.first_collection_date) >= cast(? as date) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st new file mode 100644 index 00000000..b4002ce4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st @@ -0,0 +1,38 @@ +SELECT + s.id AS "id", + s.officialname AS "officialName", + s.englishname AS "englishName", + s._typology_to_remove_ AS "typology", + s.eosc_type AS "eoscType", + s.eosc_datasource_type AS "eoscDatasourceType", + s.registeredby AS "registeredBy", + s.registrationdate::text AS "registrationDate", + MIN(a.first_collection_date) AS "firstCollectionDate", + MAX(a.last_collection_date) AS "lastCollectionDate", + (array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal", + array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities", + array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations" +FROM + dsm_services s + left outer join dsm_api a on (s.id = a.service) + left outer join dsm_service_organization dso on (s.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +WHERE + s.registrationdate is not null + and s.registeredby is not null + and s.managed = true +GROUP BY + s.id, + s.officialname, + s.englishname, + s._typology_to_remove_, + s.eosc_type, + s.eosc_datasource_type, + s.registeredby, + s.registrationdate +HAVING + s.registrationdate < max(a.last_collection_date) + and sum(a.last_collection_total) > 0 +ORDER BY s.registrationdate desc +LIMIT ? +