From 6965742b778826a6b2c5a97ca3ebe5719d2592f5 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 26 Jun 2024 11:23:09 +0200 Subject: [PATCH] funders api: remove history from mongo, remove temporary file, use of the last_aggregation_date field in dsm_service_funder --- .../openaire/funders/FunderService.java | 134 ++---------------- .../funders/FundersApiController.java | 62 ++++---- .../funders/domain/db/FunderDbEntry.java | 11 +- ...{FunderUpdate.java => FunderDbUpdate.java} | 2 +- .../src/main/resources/sql/funders-schema.sql | 11 +- 5 files changed, 51 insertions(+), 169 deletions(-) rename apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/{FunderUpdate.java => FunderDbUpdate.java} (96%) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index d7092b9e..83e1f5e6 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -1,49 +1,27 @@ package eu.dnetlib.openaire.funders; -import java.io.File; -import java.io.FileWriter; -import java.io.FilenameFilter; -import java.io.IOException; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import javax.annotation.PostConstruct; - import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - -import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; -import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; -import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate; import eu.dnetlib.openaire.funders.domain.db.FunderPid; -import eu.dnetlib.openaire.funders.domain.db.FunderUpdate; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") public class FunderService { - private static final String TEMP_FILE_SUFFIX = ".funds.tmp"; private static final String SEPARATOR = "@=@"; @Autowired @@ -52,81 +30,23 @@ public class FunderService { @Autowired private JdbcTemplate jdbcTemplate; - @Autowired - private MongoLoggerClient mongoLoggerClient; - - private File tempDir; - - private File tempFile; - - private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - private static final Log log = LogFactory.getLog(FunderService.class); - - @PostConstruct - public void init() { - - tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp")); - - for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) { - deleteFile(f); - } - - new Thread(this::updateFunders).start(); - } - - private void deleteFile(final File f) { - if (f != null && f.exists()) { - log.info("Deleting file: " + f.getAbsolutePath()); - f.delete(); - } - } - - @Scheduled(cron = "${openaire.exporter.funders.cron}") - public void updateFunders() { - final ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new JavaTimeModule()); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - - final String records = StreamSupport.stream(funderRepository.findAll().spliterator(), false) + public List getFunders() { + return StreamSupport.stream(funderRepository.findAll().spliterator(), false) .map(this::patchFunder) - .map(this::addAggregationHistory) - .map(funder -> { - try { - return mapper.writeValueAsString(funder); - } catch (final JsonProcessingException e) { - log.error("Error generating funders file", e); - throw new RuntimeException("Error generating funders file", e); - } - }) - .collect(Collectors.joining(",")); + .collect(Collectors.toList()); - try { - final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir); - log.info("Generating funders file: " + tmp.getAbsolutePath()); - - try (final FileWriter writer = new FileWriter(tmp)) { - writer.write("["); - writer.write(records); - writer.write("]"); - log.info("Publish funders file: " + tmp.getAbsolutePath()); - - deleteFile(tempFile); - setTempFile(tmp); - } - } catch (final IOException e) { - log.error("Error generating funders file", e); - throw new RuntimeException("Error generating funders file", e); - } } public FunderDbEntry getFunder(final String id) throws FundersApiException { return funderRepository.findById(id) .map(this::patchFunder) - .map(this::addAggregationHistory) .orElseThrow(() -> new FundersApiException("Missing Funder: " + id)); } + public boolean isValidFunder(final String id) { + return funderRepository.existsById(id); + } + private FunderDbEntry patchFunder(final FunderDbEntry funder) { // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) @@ -163,35 +83,7 @@ public class FunderService { return funder; } - private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) { - - final List dates = funder.getDatasources() - .stream() - .map(FunderDatasource::getId) - .map(id -> { - try { - return mongoLoggerClient.getAggregationHistoryV2(id); - } catch (final DsmApiException e) { - log.error("Error retrieving the aggregation history", e); - throw new RuntimeException("Error retrieving the aggregation history", e); - } - }) - .flatMap(List::stream) - .filter(AggregationInfo::isCompletedSuccessfully) - .filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) - .map(AggregationInfo::getDate) - .distinct() - .map(s -> LocalDate.parse(s, DATEFORMATTER)) - .sorted(Comparator.reverseOrder()) - .limit(10) - .collect(Collectors.toList()); - - funder.setAggregationDates(dates); - - return funder; - } - - public void updateFunder(final String id, final FunderUpdate funderUpdate) { + public void updateFunder(final String id, final FunderDbUpdate funderUpdate) { final String sql = "UPDATE dsm_organizations SET (" @@ -232,12 +124,4 @@ public class FunderService { } - public File getTempFile() { - return tempFile; - } - - public void setTempFile(final File tempFile) { - this.tempFile = tempFile; - } - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index 7f74f94f..436cc736 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -1,30 +1,22 @@ package eu.dnetlib.openaire.funders; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.OutputStream; +import java.util.List; -import javax.servlet.http.HttpServletResponse; - -import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; -import eu.dnetlib.openaire.funders.domain.db.FunderUpdate; +import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -51,28 +43,29 @@ public class FundersApiController extends AbstractExporterController { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public void getFunders(final HttpServletResponse res) throws FundersApiException { - - res.setContentType(MediaType.APPLICATION_JSON_VALUE); - - final File file = service.getTempFile(); - - if (file == null) { - log.error("Missing temp file (NULL)"); - throw new FundersApiException("Missing temp file (NULL)"); + public List getFunders() throws FundersApiException { + try { + return service.getFunders(); + } catch (final Throwable e) { + log.error("Error getting funders", e); + throw e; } + } - if (!file.exists()) { - log.error("Missing temp file " + service.getTempFile()); - throw new FundersApiException("Missing temp file " + service.getTempFile()); - } - - try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) { - IOUtils.copy(in, out); - return; - } catch (final Exception e) { - log.error("Error reading file " + service.getTempFile(), e); - throw new FundersApiException("Error reading file " + service.getTempFile(), e); + @RequestMapping(value = "/funders/{id}", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "get a funder by Id", description = "get a funder by Id") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public FunderDbEntry getFunder(@PathVariable final String id) throws FundersApiException { + try { + return service.getFunder(id); + } catch (final Throwable e) { + log.error("Error getting funder: " + id, e); + throw e; } } @@ -84,9 +77,12 @@ public class FundersApiController extends AbstractExporterController { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public @ResponseBody FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderUpdate funderUpdate) throws FundersApiException { - service.updateFunder(id, funderUpdate); - return service.getFunder(id); + public FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderDbUpdate funderUpdate) throws FundersApiException { + if (service.isValidFunder(id)) { + service.updateFunder(id, funderUpdate); + return service.getFunder(id); + } + throw new FundersApiException("Invalid funder: " + id); } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index 1a3219c8..b3af6e00 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -21,7 +21,7 @@ import com.vladmihalcea.hibernate.type.array.StringArrayType; @Entity @Table(name = "funders_view") @TypeDefs({ - @TypeDef(name = "string-array", typeClass = StringArrayType.class) + @TypeDef(name = "string-array", typeClass = StringArrayType.class), }) public class FunderDbEntry implements Serializable { @@ -68,8 +68,9 @@ public class FunderDbEntry implements Serializable { @Transient private List datasources = new ArrayList(); - @Transient - private List aggregationDates; + @Type(type = "string-array") + @Column(name = "aggregationdates", columnDefinition = "text[]") + private String[] aggregationDates; public String getId() { return id; @@ -167,11 +168,11 @@ public class FunderDbEntry implements Serializable { this.datasources = datasources; } - public List getAggregationDates() { + public String[] getAggregationDates() { return aggregationDates; } - public void setAggregationDates(final List aggregationDates) { + public void setAggregationDates(final String[] aggregationDates) { this.aggregationDates = aggregationDates; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java similarity index 96% rename from apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java rename to apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java index 666ad304..8356aca7 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java @@ -4,7 +4,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; -public class FunderUpdate implements Serializable { +public class FunderDbUpdate implements Serializable { private static final long serialVersionUID = -9086478785780647627L; diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 733ee598..0b198bdf 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -5,16 +5,16 @@ CREATE TABLE dsm_service_funder ( _dnet_resource_identifier_ varchar(2048) DEFAULT 'temp_'||md5(clock_timestamp()::text)||'_'||md5(random()::text), service text NOT NULL REFERENCES dsm_services(id) ON DELETE CASCADE, funder text NOT NULL REFERENCES dsm_organizations(id) ON DELETE CASCADE, - last_collection_date date, + last_aggregation_date date, PRIMARY KEY(funder, service) ); -INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_collection_date) +INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_aggregation_date) SELECT o.id||'@@'||s.id AS _dnet_resource_identifier_, s.id AS service, o.id AS funder, - max(a.last_collection_date::date) AS last_collection_date + max(a.last_aggregation_date::date) AS last_aggregation_date FROM dsm_organizations o JOIN dsm_service_organization so ON (o.id = so.organization) @@ -32,8 +32,9 @@ CREATE OR REPLACE VIEW funders_view AS SELECT o.country AS country, o.dateofcollection AS registrationdate, o.registered_funder AS registered, - array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources, - array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids + array_remove(array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type), NULL) AS datasources, + array_remove(array_agg(DISTINCT sf.last_aggregation_date::text ORDER BY sf.last_aggregation_date::text DESC), NULL) AS aggregationdates, + array_remove(array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid), NULL) AS pids FROM dsm_organizations o JOIN dsm_service_funder sf ON (o.id = sf.funder)