D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new dsm api for first collection date and registration date

This commit is contained in:
Michele Artini 2022-10-27 09:36:45 +02:00
parent 13e9821940
commit 362ab547cb
9 changed files with 427 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java
View File

@ -28,6 +28,7 @@ import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.common.OperationManager;
import eu.dnetlib.openaire.dsm.dao.ResponseUtils;
import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse;
import eu.dnetlib.openaire.dsm.domain.ApiDetails;
import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse;
@ -36,10 +37,12 @@ import eu.dnetlib.openaire.dsm.domain.DatasourceDetails;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate;
import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis;
import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse;
import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.Response;
import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.vocabularies.Country;
import io.swagger.v3.oas.annotations.Operation;
@ -163,9 +166,9 @@ public class DsmApiController extends AbstractExporterController {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public SimpleResponse<?> recentRegistered(@PathVariable final int size) throws Throwable {
public SimpleResponse<RegisteredDatasourceInfo> recentRegistered(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<?> rsp = dsmCore.searchRecentRegistered(size);
final SimpleResponse<RegisteredDatasourceInfo> rsp = dsmCore.searchRecentRegistered(size);
return prepareResponse(1, size, stop, rsp);
}
@ -422,4 +425,60 @@ public class DsmApiController extends AbstractExporterController {
.setSize(size);
return rsp;
}
// ------------------------------
@RequestMapping(value = "/ds/recentregistered/v2/{size}", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = {
DS,
R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public SimpleResponse<SimpleDatasourceInfo> recentRegisteredV2(@PathVariable final int size) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final SimpleResponse<SimpleDatasourceInfo> rsp = dsmCore.searchRecentRegisteredV2(size);
return prepareResponse(1, size, stop, rsp);
}
@RequestMapping(value = "/ds/countfirstcollect", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = {
DS,
R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public Long countFirstCollectAfter(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
return dsmCore.countFirstCollect(fromDate, typologyFilter);
}
@RequestMapping(value = "/ds/firstCollected", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = {
DS,
R
})
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public SimpleResponse<SimpleDatasourceInfo> firstCollectedAfter(@RequestParam final String fromDate,
@RequestParam(required = false) final String typologyFilter) throws Throwable {
final StopWatch stop = StopWatch.createStarted();
final List<SimpleDatasourceInfo> list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter);
final SimpleResponse<SimpleDatasourceInfo> rsp = ResponseUtils.simpleResponse(list);
return prepareResponse(1, list.size(), stop, rsp);
}
}

123
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java
View File

@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.domain.Page;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Component;
import eu.dnetlib.enabling.datasources.common.AggregationInfo;
@ -51,6 +52,7 @@ import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.RequestFilter;
import eu.dnetlib.openaire.dsm.domain.RequestSort;
import eu.dnetlib.openaire.dsm.domain.RequestSortOrder;
import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo;
import eu.dnetlib.openaire.dsm.domain.SimpleResponse;
import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry;
import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry;
@ -275,7 +277,7 @@ public class DsmCore {
// HELPERS //////////////
public SimpleResponse<?> searchRecentRegistered(final int size) throws Throwable {
public SimpleResponse<RegisteredDatasourceInfo> searchRecentRegistered(final int size) throws Throwable {
try {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset());
@ -318,4 +320,123 @@ public class DsmCore {
return rsp;
}
public SimpleResponse<SimpleDatasourceInfo> searchRecentRegisteredV2(final int size) throws Throwable {
try {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st"), Charset.defaultCharset());
final List<SimpleDatasourceInfo> list =
jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), size);
return ResponseUtils.simpleResponse(list);
} catch (final Throwable e) {
log.error("error searching recent datasources", e);
throw e;
}
}
public Long countFirstCollect(final String fromDate, final String typeFilter) throws Throwable {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, typeFilter + "%", fromDate);
} else {
final String sql =
IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.queryForObject(sql, Long.class, fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw e;
}
}
public List<SimpleDatasourceInfo> getFirstCollectedAfter(final String fromDate, final String typeFilter) throws Throwable {
try {
if (StringUtils.isNotBlank(typeFilter)) {
final String sql =
IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate);
} else {
final String sql =
IOUtils
.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql"), Charset
.defaultCharset());
return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), fromDate);
}
} catch (final Throwable e) {
log.error("error searching datasources using the first collection date", e);
throw e;
}
}
private RowMapper<SimpleDatasourceInfo> rowMapperForSimpleDatasourceInfo() {
return (rs, rowNum) -> {
final SimpleDatasourceInfo info = new SimpleDatasourceInfo();
info.setId(rs.getString("id"));
info.setOfficialName(rs.getString("officialName"));
info.setEnglishName(rs.getString("englishName"));
info.setTypology(rs.getString("typology"));
info.setEoscType(rs.getString("eoscType"));
info.setEoscDatasourceType(rs.getString("eoscDatasourceType"));
info.setRegisteredBy(rs.getString("registeredBy"));
info.setRegistrationDate(rs.getString("registrationDate"));
info.setFirstCollectionDate(rs.getString("firstCollectionDate"));
info.setLastCollectionDate(rs.getString("lastCollectionDate"));
info.setLastCollectionTotal(rs.getLong("lastCollectionTotal"));
final Set<String> compatibilities = new HashSet<>();
for (final String s : (String[]) rs.getArray("compatibilities").getArray()) {
compatibilities.add(s);
}
// The order of the condition is important
if (compatibilities.contains("openaire-cris_1.1")) {
info.setCompatibility("openaire-cris_1.1");
} else if (compatibilities.contains("openaire4.0")) {
info.setCompatibility("openaire4.0");
} else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) {
info.setCompatibility("driver-openaire2.0");
} else if (compatibilities.contains("driver")) {
info.setCompatibility("driver");
} else if (compatibilities.contains("openaire2.0")) {
info.setCompatibility("openaire2.0");
} else if (compatibilities.contains("openaire3.0")) {
info.setCompatibility("openaire3.0");
} else if (compatibilities.contains("openaire2.0_data")) {
info.setCompatibility("openaire2.0_data");
} else if (compatibilities.contains("native")) {
info.setCompatibility("native");
} else if (compatibilities.contains("hostedBy")) {
info.setCompatibility("hostedBy");
} else if (compatibilities.contains("notCompatible")) {
info.setCompatibility("notCompatible");
} else {
info.setCompatibility("UNKNOWN");
}
for (final String s : (String[]) rs.getArray("organizations").getArray()) {
info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim());
}
return info;
};
}
}

4
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java
View File

@ -52,8 +52,8 @@ public class ResponseUtils {
return header(Lists.newLinkedList(), total);
}
public static SimpleResponse<?> simpleResponse(final List<?> list) {
final SimpleResponse rsp = new SimpleResponse().setResponse(list);;
public static <T> SimpleResponse<T> simpleResponse(final List<T> list) {
final SimpleResponse<T> rsp = new SimpleResponse<T>().setResponse(list);;
rsp.setHeader(header(Lists.newLinkedList(), list.size()));
return rsp;
}

129
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java Normal file
View File

@ -0,0 +1,129 @@
package eu.dnetlib.openaire.dsm.domain;
import java.util.LinkedHashMap;
import java.util.Map;
public class SimpleDatasourceInfo {
private String id;
private String officialName;
private String englishName;
private Map<String, String> organizations = new LinkedHashMap<>();
@Deprecated
private String typology;
private String eoscType;
private String eoscDatasourceType;
private String registeredBy;
private String registrationDate;
private String compatibility;
private String firstCollectionDate;
private String lastCollectionDate;
private long lastCollectionTotal;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOfficialName() {
return officialName;
}
public void setOfficialName(final String officialName) {
this.officialName = officialName;
}
public String getEnglishName() {
return englishName;
}
public void setEnglishName(final String englishName) {
this.englishName = englishName;
}
public Map<String, String> getOrganizations() {
return organizations;
}
public void setOrganizations(final Map<String, String> organizations) {
this.organizations = organizations;
}
@Deprecated
public String getTypology() {
return typology;
}
@Deprecated
public void setTypology(final String typology) {
this.typology = typology;
}
public String getEoscType() {
return eoscType;
}
public void setEoscType(final String eoscType) {
this.eoscType = eoscType;
}
public String getEoscDatasourceType() {
return eoscDatasourceType;
}
public void setEoscDatasourceType(final String eoscDatasourceType) {
this.eoscDatasourceType = eoscDatasourceType;
}
public String getRegisteredBy() {
return registeredBy;
}
public void setRegisteredBy(final String registeredBy) {
this.registeredBy = registeredBy;
}
public String getRegistrationDate() {
return registrationDate;
}
public void setRegistrationDate(final String registrationDate) {
this.registrationDate = registrationDate;
}
public String getCompatibility() {
return compatibility;
}
public void setCompatibility(final String compatibility) {
this.compatibility = compatibility;
}
public String getFirstCollectionDate() {
return firstCollectionDate;
}
public void setFirstCollectionDate(final String firstCollectionDate) {
this.firstCollectionDate = firstCollectionDate;
}
public String getLastCollectionDate() {
return lastCollectionDate;
}
public void setLastCollectionDate(final String lastCollectionDate) {
this.lastCollectionDate = lastCollectionDate;
}
public long getLastCollectionTotal() {
return lastCollectionTotal;
}
public void setLastCollectionTotal(final long lastCollectionTotal) {
this.lastCollectionTotal = lastCollectionTotal;
}
}

7
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql Normal file
View File

@ -0,0 +1,7 @@
select count(*) as count
from (
select d.id
from dsm_services d left outer join dsm_api a on (d.id = a.service)
group by d.id
having min(a.first_collection_date) >= cast(? as date)
) as t

8
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql Normal file
View File

@ -0,0 +1,8 @@
select count(*) as count
from (
select d.id
from dsm_services d left outer join dsm_api a on (d.id = a.service)
where d._typology_to_remove_ like ?
group by d.id
having min(a.first_collection_date) >= cast(? as date)
) as t

29
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql Normal file
View File

@ -0,0 +1,29 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING MIN(a.first_collection_date) >= cast(? as date)

31
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql Normal file
View File

@ -0,0 +1,31 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
WHERE
s._typology_to_remove_ like ?
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING MIN(a.first_collection_date) >= cast(? as date)

38
apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st Normal file
View File

@ -0,0 +1,38 @@
SELECT
s.id AS "id",
s.officialname AS "officialName",
s.englishname AS "englishName",
s._typology_to_remove_ AS "typology",
s.eosc_type AS "eoscType",
s.eosc_datasource_type AS "eoscDatasourceType",
s.registeredby AS "registeredBy",
s.registrationdate::text AS "registrationDate",
MIN(a.first_collection_date) AS "firstCollectionDate",
MAX(a.last_collection_date) AS "lastCollectionDate",
(array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal",
array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities",
array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations"
FROM
dsm_services s
left outer join dsm_api a on (s.id = a.service)
left outer join dsm_service_organization dso on (s.id = dso.service)
left outer join dsm_organizations o on (o.id = dso.organization)
WHERE
s.registrationdate is not null
and s.registeredby is not null
and s.managed = true
GROUP BY
s.id,
s.officialname,
s.englishname,
s._typology_to_remove_,
s.eosc_type,
s.eosc_datasource_type,
s.registeredby,
s.registrationdate
HAVING
s.registrationdate < max(a.last_collection_date)
and sum(a.last_collection_total) > 0
ORDER BY s.registrationdate desc
LIMIT ?