From fb947fde6207409f99ccb6c55179e9af4865a922 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 24 Jun 2024 15:15:47 +0200 Subject: [PATCH 1/5] funders api: created dsm_service_funder table + updated view --- .../src/main/resources/sql/funders-schema.sql | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 9e733f32..3edb232f 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -1,6 +1,15 @@ ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean; -CREATE VIEW funders_view AS SELECT + +CREATE TABLE dsm_service_funder ( + _dnet_resource_identifier_ varchar(2048) DEFAULT 'temp_'||md5(clock_timestamp()::text)||'_'||md5(random()::text), + service text NOT NULL REFERENCES dsm_services(id) ON DELETE CASCADE, + funder text NOT NULL REFERENCES dsm_organizations(id) ON DELETE CASCADE, + last_collection_date date, + PRIMARY KEY(funder, service) +); + +CREATE OR REPLACE VIEW funders_view AS SELECT o.id AS id, o.legalshortname AS legalshortname, o.legalname AS legalname, @@ -13,13 +22,13 @@ CREATE VIEW funders_view AS SELECT array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids FROM dsm_organizations o - JOIN dsm_service_organization so ON (o.id = so.organization) - JOIN dsm_services s ON (so.service = s.id) - JOIN projects p ON (p.collectedfrom = s.id) + JOIN dsm_service_funder sf ON (o.id = sf.funder) + JOIN dsm_services s ON (sf.service = s.id) LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization) LEFT OUTER JOIN dsm_identities pids ON (opids.pid = pids.pid) GROUP BY o.id; +GRANT ALL ON dsm_service_funder TO dnetapi; GRANT ALL ON funders_view TO dnetapi; From 3a5402d0aff4feb5f51279b43aff7343b919287c Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 24 Jun 2024 15:37:45 +0200 Subject: [PATCH 2/5] funders api: first population of dsm_service_funder --- .../src/main/resources/sql/funders-schema.sql | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 3edb232f..733ee598 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -9,6 +9,20 @@ CREATE TABLE dsm_service_funder ( PRIMARY KEY(funder, service) ); +INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_collection_date) +SELECT + o.id||'@@'||s.id AS _dnet_resource_identifier_, + s.id AS service, + o.id AS funder, + max(a.last_collection_date::date) AS last_collection_date +FROM + dsm_organizations o + JOIN dsm_service_organization so ON (o.id = so.organization) + JOIN dsm_services s ON (so.service = s.id) + JOIN projects p ON p.collectedfrom = s.id + LEFT OUTER JOIN dsm_api a ON (s.id = a.service) +GROUP BY s.id, o.id; + CREATE OR REPLACE VIEW funders_view AS SELECT o.id AS id, o.legalshortname AS legalshortname, @@ -30,5 +44,3 @@ GROUP BY o.id; GRANT ALL ON dsm_service_funder TO dnetapi; GRANT ALL ON funders_view TO dnetapi; - - From e060a3601c3859c3280932ef5fd68ea6d2dcab67 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 25 Jun 2024 11:44:53 +0200 Subject: [PATCH 3/5] new api ti update funders --- .../openaire/funders/FunderService.java | 170 ++++++++++++------ .../funders/FundersApiController.java | 18 ++ .../funders/domain/db/FunderUpdate.java | 74 ++++++++ 3 files changed, 207 insertions(+), 55 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index 756259c9..d7092b9e 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -3,6 +3,7 @@ package eu.dnetlib.openaire.funders; import java.io.File; import java.io.FileWriter; import java.io.FilenameFilter; +import java.io.IOException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.Arrays; @@ -10,6 +11,7 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.annotation.PostConstruct; @@ -18,20 +20,24 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; +import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; import eu.dnetlib.openaire.funders.domain.db.FunderPid; +import eu.dnetlib.openaire.funders.domain.db.FunderUpdate; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") @@ -43,6 +49,9 @@ public class FunderService { @Autowired private FunderRepository funderRepository; + @Autowired + private JdbcTemplate jdbcTemplate; + @Autowired private MongoLoggerClient mongoLoggerClient; @@ -75,78 +84,86 @@ public class FunderService { @Scheduled(cron = "${openaire.exporter.funders.cron}") public void updateFunders() { + final ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + final String records = StreamSupport.stream(funderRepository.findAll().spliterator(), false) + .map(this::patchFunder) + .map(this::addAggregationHistory) + .map(funder -> { + try { + return mapper.writeValueAsString(funder); + } catch (final JsonProcessingException e) { + log.error("Error generating funders file", e); + throw new RuntimeException("Error generating funders file", e); + } + }) + .collect(Collectors.joining(",")); + try { - final ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir); - log.info("Generating funders file: " + tmp.getAbsolutePath()); try (final FileWriter writer = new FileWriter(tmp)) { writer.write("["); - boolean first = true; - for (final FunderDbEntry funder : funderRepository.findAll()) { - log.info(" - adding: " + funder.getId()); - - // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) - - final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) - .filter(Objects::nonNull) - .map(s -> s.split(SEPARATOR)) - .filter(arr -> arr.length == 3) - .map(arr -> { - final FunderDatasource ds = new FunderDatasource(); - ds.setId(arr[0].trim()); - ds.setName(arr[1].trim()); - ds.setType(arr[2].trim()); - return ds; - }) - .filter(ds -> StringUtils.isNotBlank(ds.getId())) - .collect(Collectors.toList()); - - funder.setDatasources(datasources); - - final List pids = Arrays.stream(funder.getPidsPostgres()) - .filter(Objects::nonNull) - .map(s -> s.split(SEPARATOR)) - .filter(arr -> arr.length == 2) - .map(arr -> { - final FunderPid pid = new FunderPid(); - pid.setType(arr[0].trim()); - pid.setValue(arr[1].trim()); - return pid; - }) - .filter(pid -> StringUtils.isNotBlank(pid.getValue())) - .collect(Collectors.toList()); - - funder.setPids(pids); - - // END PATCH - - addAggregationHistory(funder); - - if (first) { - first = false; - } else { - writer.write(","); - } - writer.write(mapper.writeValueAsString(funder)); - } + writer.write(records); writer.write("]"); log.info("Publish funders file: " + tmp.getAbsolutePath()); deleteFile(tempFile); setTempFile(tmp); } - } catch (final Throwable e) { + } catch (final IOException e) { log.error("Error generating funders file", e); throw new RuntimeException("Error generating funders file", e); } } - private void addAggregationHistory(final FunderDbEntry funder) { + public FunderDbEntry getFunder(final String id) throws FundersApiException { + return funderRepository.findById(id) + .map(this::patchFunder) + .map(this::addAggregationHistory) + .orElseThrow(() -> new FundersApiException("Missing Funder: " + id)); + } + + private FunderDbEntry patchFunder(final FunderDbEntry funder) { + // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) + final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 3) + .map(arr -> { + final FunderDatasource ds = new FunderDatasource(); + ds.setId(arr[0].trim()); + ds.setName(arr[1].trim()); + ds.setType(arr[2].trim()); + return ds; + }) + .filter(ds -> StringUtils.isNotBlank(ds.getId())) + .collect(Collectors.toList()); + + funder.setDatasources(datasources); + + final List pids = Arrays.stream(funder.getPidsPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 2) + .map(arr -> { + final FunderPid pid = new FunderPid(); + pid.setType(arr[0].trim()); + pid.setValue(arr[1].trim()); + return pid; + }) + .filter(pid -> StringUtils.isNotBlank(pid.getValue())) + .collect(Collectors.toList()); + + funder.setPids(pids); + + return funder; + } + + private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) { final List dates = funder.getDatasources() .stream() @@ -170,6 +187,49 @@ public class FunderService { .collect(Collectors.toList()); funder.setAggregationDates(dates); + + return funder; + } + + public void updateFunder(final String id, final FunderUpdate funderUpdate) { + + final String sql = + "UPDATE dsm_organizations SET (" + + " legalshortname," + + " legalname," + + " websiteurl," + + " logourl," + + " country," + + " registered_funder" + + ") = (" + + " coalesce(?, legalshortname)," + + " coalesce(?, legalname)," + + " coalesce(?, websiteurl)," + + " coalesce(?, logourl)," + + " coalesce(?, country)," + + " coalesce(?, registered_funder)" + + ") WHERE id = ?"; + + jdbcTemplate.update(sql, funderUpdate.getLegalShortName(), funderUpdate.getLegalName(), funderUpdate.getWebsiteUrl(), funderUpdate + .getLogoUrl(), funderUpdate.getCountry(), funderUpdate.getRegistered(), id); + + if (funderUpdate.getPids() != null) { + funderUpdate.getPids().forEach(pid -> { + // TODO: the first update should be deleted after the re-implementation of the pid-tables, + // TODO: the field 'type' should also be moved in the second update + + if (jdbcTemplate.queryForObject("SELECT count(*) FROM dsm_identities WHERE issuertype = ? AND pid = ?", Integer.class, pid.getType(), pid + .getValue()) == 0) { + jdbcTemplate.update("INSERT INTO dsm_identities(issuertype, pid) VALUES (?, ?)", pid.getType(), pid.getValue()); + } + + if (jdbcTemplate.queryForObject("SELECT count(*) FROM dsm_organizationpids WHERE organization = ? AND pid = ?", Integer.class, id, pid + .getValue()) == 0) { + jdbcTemplate.update("INSERT INTO dsm_organizationpids(organization, pid) VALUES (?, ?)", id, pid.getValue()); + } + }); + } + } public File getTempFile() { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index f4b0bf8d..7f74f94f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -14,12 +14,17 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FunderUpdate; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -71,4 +76,17 @@ public class FundersApiController extends AbstractExporterController { } } + @RequestMapping(value = "/funders/{id}", produces = { + "application/json" + }, method = RequestMethod.POST) + @Operation(summary = "update a funder by Id", description = "update a funder by Id") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public @ResponseBody FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderUpdate funderUpdate) throws FundersApiException { + service.updateFunder(id, funderUpdate); + return service.getFunder(id); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java new file mode 100644 index 00000000..666ad304 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java @@ -0,0 +1,74 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +public class FunderUpdate implements Serializable { + + private static final long serialVersionUID = -9086478785780647627L; + + private String legalShortName; + private String legalName; + private String websiteUrl; + private String logoUrl; + private String country; + private Boolean registered; + private List pids = new ArrayList(); + + public String getLegalShortName() { + return legalShortName; + } + + public void setLegalShortName(final String legalShortName) { + this.legalShortName = legalShortName; + } + + public String getLegalName() { + return legalName; + } + + public void setLegalName(final String legalName) { + this.legalName = legalName; + } + + public String getWebsiteUrl() { + return websiteUrl; + } + + public void setWebsiteUrl(final String websiteUrl) { + this.websiteUrl = websiteUrl; + } + + public String getLogoUrl() { + return logoUrl; + } + + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; + } + + public String getCountry() { + return country; + } + + public void setCountry(final String country) { + this.country = country; + } + + public Boolean getRegistered() { + return registered; + } + + public void setRegistered(final Boolean registered) { + this.registered = registered; + } + + public List getPids() { + return pids; + } + + public void setPids(final List pids) { + this.pids = pids; + } +} From 6965742b778826a6b2c5a97ca3ebe5719d2592f5 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 26 Jun 2024 11:23:09 +0200 Subject: [PATCH 4/5] funders api: remove history from mongo, remove temporary file, use of the last_aggregation_date field in dsm_service_funder --- .../openaire/funders/FunderService.java | 134 ++---------------- .../funders/FundersApiController.java | 62 ++++---- .../funders/domain/db/FunderDbEntry.java | 11 +- ...{FunderUpdate.java => FunderDbUpdate.java} | 2 +- .../src/main/resources/sql/funders-schema.sql | 11 +- 5 files changed, 51 insertions(+), 169 deletions(-) rename apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/{FunderUpdate.java => FunderDbUpdate.java} (96%) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index d7092b9e..83e1f5e6 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -1,49 +1,27 @@ package eu.dnetlib.openaire.funders; -import java.io.File; -import java.io.FileWriter; -import java.io.FilenameFilter; -import java.io.IOException; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import javax.annotation.PostConstruct; - import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - -import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; -import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; -import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate; import eu.dnetlib.openaire.funders.domain.db.FunderPid; -import eu.dnetlib.openaire.funders.domain.db.FunderUpdate; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") public class FunderService { - private static final String TEMP_FILE_SUFFIX = ".funds.tmp"; private static final String SEPARATOR = "@=@"; @Autowired @@ -52,81 +30,23 @@ public class FunderService { @Autowired private JdbcTemplate jdbcTemplate; - @Autowired - private MongoLoggerClient mongoLoggerClient; - - private File tempDir; - - private File tempFile; - - private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - private static final Log log = LogFactory.getLog(FunderService.class); - - @PostConstruct - public void init() { - - tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp")); - - for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) { - deleteFile(f); - } - - new Thread(this::updateFunders).start(); - } - - private void deleteFile(final File f) { - if (f != null && f.exists()) { - log.info("Deleting file: " + f.getAbsolutePath()); - f.delete(); - } - } - - @Scheduled(cron = "${openaire.exporter.funders.cron}") - public void updateFunders() { - final ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - - final String records = StreamSupport.stream(funderRepository.findAll().spliterator(), false) + public List getFunders() { + return StreamSupport.stream(funderRepository.findAll().spliterator(), false) .map(this::patchFunder) - .map(this::addAggregationHistory) - .map(funder -> { - try { - return mapper.writeValueAsString(funder); - } catch (final JsonProcessingException e) { - log.error("Error generating funders file", e); - throw new RuntimeException("Error generating funders file", e); - } - }) - .collect(Collectors.joining(",")); + .collect(Collectors.toList()); - try { - final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir); - log.info("Generating funders file: " + tmp.getAbsolutePath()); - - try (final FileWriter writer = new FileWriter(tmp)) { - writer.write("["); - writer.write(records); - writer.write("]"); - log.info("Publish funders file: " + tmp.getAbsolutePath()); - - deleteFile(tempFile); - setTempFile(tmp); - } - } catch (final IOException e) { - log.error("Error generating funders file", e); - throw new RuntimeException("Error generating funders file", e); - } } public FunderDbEntry getFunder(final String id) throws FundersApiException { return funderRepository.findById(id) .map(this::patchFunder) - .map(this::addAggregationHistory) .orElseThrow(() -> new FundersApiException("Missing Funder: " + id)); } + public boolean isValidFunder(final String id) { + return funderRepository.existsById(id); + } + private FunderDbEntry patchFunder(final FunderDbEntry funder) { // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) @@ -163,35 +83,7 @@ public class FunderService { return funder; } - private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) { - - final List dates = funder.getDatasources() - .stream() - .map(FunderDatasource::getId) - .map(id -> { - try { - return mongoLoggerClient.getAggregationHistoryV2(id); - } catch (final DsmApiException e) { - log.error("Error retrieving the aggregation history", e); - throw new RuntimeException("Error retrieving the aggregation history", e); - } - }) - .flatMap(List::stream) - .filter(AggregationInfo::isCompletedSuccessfully) - .filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) - .map(AggregationInfo::getDate) - .distinct() - .map(s -> LocalDate.parse(s, DATEFORMATTER)) - .sorted(Comparator.reverseOrder()) - .limit(10) - .collect(Collectors.toList()); - - funder.setAggregationDates(dates); - - return funder; - } - - public void updateFunder(final String id, final FunderUpdate funderUpdate) { + public void updateFunder(final String id, final FunderDbUpdate funderUpdate) { final String sql = "UPDATE dsm_organizations SET (" @@ -232,12 +124,4 @@ public class FunderService { } - public File getTempFile() { - return tempFile; - } - - public void setTempFile(final File tempFile) { - this.tempFile = tempFile; - } - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index 7f74f94f..436cc736 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -1,30 +1,22 @@ package eu.dnetlib.openaire.funders; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.OutputStream; +import java.util.List; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; -import eu.dnetlib.openaire.funders.domain.db.FunderUpdate; +import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -51,28 +43,29 @@ public class FundersApiController extends AbstractExporterController { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public void getFunders(final HttpServletResponse res) throws FundersApiException { - - res.setContentType(MediaType.APPLICATION_JSON_VALUE); - - final File file = service.getTempFile(); - - if (file == null) { - log.error("Missing temp file (NULL)"); - throw new FundersApiException("Missing temp file (NULL)"); + public List getFunders() throws FundersApiException { + try { + return service.getFunders(); + } catch (final Throwable e) { + log.error("Error getting funders", e); + throw e; } + } - if (!file.exists()) { - log.error("Missing temp file " + service.getTempFile()); - throw new FundersApiException("Missing temp file " + service.getTempFile()); - } - - try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) { - IOUtils.copy(in, out); - return; - } catch (final Exception e) { - log.error("Error reading file " + service.getTempFile(), e); - throw new FundersApiException("Error reading file " + service.getTempFile(), e); + @RequestMapping(value = "/funders/{id}", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "get a funder by Id", description = "get a funder by Id") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public FunderDbEntry getFunder(@PathVariable final String id) throws FundersApiException { + try { + return service.getFunder(id); + } catch (final Throwable e) { + log.error("Error getting funder: " + id, e); + throw e; } } @@ -84,9 +77,12 @@ public class FundersApiController extends AbstractExporterController { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public @ResponseBody FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderUpdate funderUpdate) throws FundersApiException { - service.updateFunder(id, funderUpdate); - return service.getFunder(id); + public FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderDbUpdate funderUpdate) throws FundersApiException { + if (service.isValidFunder(id)) { + service.updateFunder(id, funderUpdate); + return service.getFunder(id); + } + throw new FundersApiException("Invalid funder: " + id); } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index 1a3219c8..b3af6e00 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -21,7 +21,7 @@ import com.vladmihalcea.hibernate.type.array.StringArrayType; @Entity @Table(name = "funders_view") @TypeDefs({ - @TypeDef(name = "string-array", typeClass = StringArrayType.class) + @TypeDef(name = "string-array", typeClass = StringArrayType.class), }) public class FunderDbEntry implements Serializable { @@ -68,8 +68,9 @@ public class FunderDbEntry implements Serializable { @Transient private List datasources = new ArrayList(); - @Transient - private List aggregationDates; + @Type(type = "string-array") + @Column(name = "aggregationdates", columnDefinition = "text[]") + private String[] aggregationDates; public String getId() { return id; @@ -167,11 +168,11 @@ public class FunderDbEntry implements Serializable { this.datasources = datasources; } - public List getAggregationDates() { + public String[] getAggregationDates() { return aggregationDates; } - public void setAggregationDates(final List aggregationDates) { + public void setAggregationDates(final String[] aggregationDates) { this.aggregationDates = aggregationDates; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java similarity index 96% rename from apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java rename to apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java index 666ad304..8356aca7 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java @@ -4,7 +4,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; -public class FunderUpdate implements Serializable { +public class FunderDbUpdate implements Serializable { private static final long serialVersionUID = -9086478785780647627L; diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 733ee598..0b198bdf 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -5,16 +5,16 @@ CREATE TABLE dsm_service_funder ( _dnet_resource_identifier_ varchar(2048) DEFAULT 'temp_'||md5(clock_timestamp()::text)||'_'||md5(random()::text), service text NOT NULL REFERENCES dsm_services(id) ON DELETE CASCADE, funder text NOT NULL REFERENCES dsm_organizations(id) ON DELETE CASCADE, - last_collection_date date, + last_aggregation_date date, PRIMARY KEY(funder, service) ); -INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_collection_date) +INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_aggregation_date) SELECT o.id||'@@'||s.id AS _dnet_resource_identifier_, s.id AS service, o.id AS funder, - max(a.last_collection_date::date) AS last_collection_date + max(a.last_aggregation_date::date) AS last_aggregation_date FROM dsm_organizations o JOIN dsm_service_organization so ON (o.id = so.organization) @@ -32,8 +32,9 @@ CREATE OR REPLACE VIEW funders_view AS SELECT o.country AS country, o.dateofcollection AS registrationdate, o.registered_funder AS registered, - array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources, - array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids + array_remove(array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type), NULL) AS datasources, + array_remove(array_agg(DISTINCT sf.last_aggregation_date::text ORDER BY sf.last_aggregation_date::text DESC), NULL) AS aggregationdates, + array_remove(array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid), NULL) AS pids FROM dsm_organizations o JOIN dsm_service_funder sf ON (o.id = sf.funder) From 84bdffc3f43411226baeebf4152fc48e9db132a3 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 26 Jun 2024 11:44:11 +0200 Subject: [PATCH 5/5] use of sql dates --- .../funders/domain/db/FunderDbEntry.java | 20 ++++++++++--------- .../src/main/resources/sql/funders-schema.sql | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index b3af6e00..e6640821 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -1,7 +1,7 @@ package eu.dnetlib.openaire.funders.domain.db; import java.io.Serializable; -import java.time.LocalDate; +import java.sql.Date; import java.util.ArrayList; import java.util.List; @@ -16,12 +16,14 @@ import org.hibernate.annotations.TypeDef; import org.hibernate.annotations.TypeDefs; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.vladmihalcea.hibernate.type.array.DateArrayType; import com.vladmihalcea.hibernate.type.array.StringArrayType; @Entity @Table(name = "funders_view") @TypeDefs({ @TypeDef(name = "string-array", typeClass = StringArrayType.class), + @TypeDef(name = "date-array", typeClass = DateArrayType.class), }) public class FunderDbEntry implements Serializable { @@ -47,7 +49,7 @@ public class FunderDbEntry implements Serializable { private String country; @Column(name = "registrationdate") - private LocalDate registrationDate; + private Date registrationDate; @Column(name = "registered") private Boolean registered; @@ -68,9 +70,9 @@ public class FunderDbEntry implements Serializable { @Transient private List datasources = new ArrayList(); - @Type(type = "string-array") - @Column(name = "aggregationdates", columnDefinition = "text[]") - private String[] aggregationDates; + @Type(type = "date-array") + @Column(name = "aggregationdates", columnDefinition = "date[]") + private Date[] aggregationDates; public String getId() { return id; @@ -120,11 +122,11 @@ public class FunderDbEntry implements Serializable { this.country = country; } - public LocalDate getRegistrationDate() { + public Date getRegistrationDate() { return registrationDate; } - public void setRegistrationDate(final LocalDate registrationDate) { + public void setRegistrationDate(final Date registrationDate) { this.registrationDate = registrationDate; } @@ -168,11 +170,11 @@ public class FunderDbEntry implements Serializable { this.datasources = datasources; } - public String[] getAggregationDates() { + public Date[] getAggregationDates() { return aggregationDates; } - public void setAggregationDates(final String[] aggregationDates) { + public void setAggregationDates(final Date[] aggregationDates) { this.aggregationDates = aggregationDates; } diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 0b198bdf..9c3c5f1b 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -33,7 +33,7 @@ CREATE OR REPLACE VIEW funders_view AS SELECT o.dateofcollection AS registrationdate, o.registered_funder AS registered, array_remove(array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type), NULL) AS datasources, - array_remove(array_agg(DISTINCT sf.last_aggregation_date::text ORDER BY sf.last_aggregation_date::text DESC), NULL) AS aggregationdates, + array_remove(array_agg(DISTINCT sf.last_aggregation_date ORDER BY sf.last_aggregation_date DESC), NULL) AS aggregationdates, array_remove(array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid), NULL) AS pids FROM dsm_organizations o