D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

funders api: remove history from mongo, remove temporary file, use of the last_aggregation_date field in dsm_service_funder

This commit is contained in:
Michele Artini 2024-06-26 11:23:09 +02:00
parent e060a3601c
commit 6965742b77
5 changed files with 51 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java
View File

@ -1,49 +1,27 @@
package eu.dnetlib.openaire.funders;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate;
import eu.dnetlib.openaire.funders.domain.db.FunderPid;
import eu.dnetlib.openaire.funders.domain.db.FunderUpdate;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public class FunderService {
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
private static final String SEPARATOR = "@=@";
@Autowired
@ -52,81 +30,23 @@ public class FunderService {
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
private MongoLoggerClient mongoLoggerClient;
private File tempDir;
private File tempFile;
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Log log = LogFactory.getLog(FunderService.class);
@PostConstruct
public void init() {
tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp"));
for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
deleteFile(f);
}
new Thread(this::updateFunders).start();
}
private void deleteFile(final File f) {
if (f != null && f.exists()) {
log.info("Deleting file: " + f.getAbsolutePath());
f.delete();
}
}
@Scheduled(cron = "${openaire.exporter.funders.cron}")
public void updateFunders() {
final ObjectMapper mapper = new ObjectMapper();
mapper.registerModule(new JavaTimeModule());
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
final String records = StreamSupport.stream(funderRepository.findAll().spliterator(), false)
public List<FunderDbEntry> getFunders() {
return StreamSupport.stream(funderRepository.findAll().spliterator(), false)
.map(this::patchFunder)
.map(this::addAggregationHistory)
.map(funder -> {
try {
return mapper.writeValueAsString(funder);
} catch (final JsonProcessingException e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
})
.collect(Collectors.joining(","));
.collect(Collectors.toList());
try {
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
log.info("Generating funders file: " + tmp.getAbsolutePath());
try (final FileWriter writer = new FileWriter(tmp)) {
writer.write("[");
writer.write(records);
writer.write("]");
log.info("Publish funders file: " + tmp.getAbsolutePath());
deleteFile(tempFile);
setTempFile(tmp);
}
} catch (final IOException e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
}
public FunderDbEntry getFunder(final String id) throws FundersApiException {
return funderRepository.findById(id)
.map(this::patchFunder)
.map(this::addAggregationHistory)
.orElseThrow(() -> new FundersApiException("Missing Funder: " + id));
}
public boolean isValidFunder(final String id) {
return funderRepository.existsById(id);
}
private FunderDbEntry patchFunder(final FunderDbEntry funder) {
// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)
final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
@ -163,35 +83,7 @@ public class FunderService {
return funder;
}
private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) {
final List<LocalDate> dates = funder.getDatasources()
.stream()
.map(FunderDatasource::getId)
.map(id -> {
try {
return mongoLoggerClient.getAggregationHistoryV2(id);
} catch (final DsmApiException e) {
log.error("Error retrieving the aggregation history", e);
throw new RuntimeException("Error retrieving the aggregation history", e);
}
})
.flatMap(List::stream)
.filter(AggregationInfo::isCompletedSuccessfully)
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM)
.map(AggregationInfo::getDate)
.distinct()
.map(s -> LocalDate.parse(s, DATEFORMATTER))
.sorted(Comparator.reverseOrder())
.limit(10)
.collect(Collectors.toList());
funder.setAggregationDates(dates);
return funder;
}
public void updateFunder(final String id, final FunderUpdate funderUpdate) {
public void updateFunder(final String id, final FunderDbUpdate funderUpdate) {
final String sql =
"UPDATE dsm_organizations SET ("
@ -232,12 +124,4 @@ public class FunderService {
}
public File getTempFile() {
return tempFile;
}
public void setTempFile(final File tempFile) {
this.tempFile = tempFile;
}
}

62
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java
View File

@ -1,30 +1,22 @@
package eu.dnetlib.openaire.funders;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderUpdate;
import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
@ -51,28 +43,29 @@ public class FundersApiController extends AbstractExporterController {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public void getFunders(final HttpServletResponse res) throws FundersApiException {
res.setContentType(MediaType.APPLICATION_JSON_VALUE);
final File file = service.getTempFile();
if (file == null) {
log.error("Missing temp file (NULL)");
throw new FundersApiException("Missing temp file (NULL)");
public List<FunderDbEntry> getFunders() throws FundersApiException {
try {
return service.getFunders();
} catch (final Throwable e) {
log.error("Error getting funders", e);
throw e;
}
}
if (!file.exists()) {
log.error("Missing temp file " + service.getTempFile());
throw new FundersApiException("Missing temp file " + service.getTempFile());
}
try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) {
IOUtils.copy(in, out);
return;
} catch (final Exception e) {
log.error("Error reading file " + service.getTempFile(), e);
throw new FundersApiException("Error reading file " + service.getTempFile(), e);
@RequestMapping(value = "/funders/{id}", produces = {
"application/json"
}, method = RequestMethod.GET)
@Operation(summary = "get a funder by Id", description = "get a funder by Id")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public FunderDbEntry getFunder(@PathVariable final String id) throws FundersApiException {
try {
return service.getFunder(id);
} catch (final Throwable e) {
log.error("Error getting funder: " + id, e);
throw e;
}
}
@ -84,9 +77,12 @@ public class FundersApiController extends AbstractExporterController {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public @ResponseBody FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderUpdate funderUpdate) throws FundersApiException {
service.updateFunder(id, funderUpdate);
return service.getFunder(id);
public FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderDbUpdate funderUpdate) throws FundersApiException {
if (service.isValidFunder(id)) {
service.updateFunder(id, funderUpdate);
return service.getFunder(id);
}
throw new FundersApiException("Invalid funder: " + id);
}
}

11
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java
View File

@ -21,7 +21,7 @@ import com.vladmihalcea.hibernate.type.array.StringArrayType;
@Entity
@Table(name = "funders_view")
@TypeDefs({
@TypeDef(name = "string-array", typeClass = StringArrayType.class)
@TypeDef(name = "string-array", typeClass = StringArrayType.class),
})
public class FunderDbEntry implements Serializable {
@ -68,8 +68,9 @@ public class FunderDbEntry implements Serializable {
@Transient
private List<FunderDatasource> datasources = new ArrayList<FunderDatasource>();
@Transient
private List<LocalDate> aggregationDates;
@Type(type = "string-array")
@Column(name = "aggregationdates", columnDefinition = "text[]")
private String[] aggregationDates;
public String getId() {
return id;
@ -167,11 +168,11 @@ public class FunderDbEntry implements Serializable {
this.datasources = datasources;
}
public List<LocalDate> getAggregationDates() {
public String[] getAggregationDates() {
return aggregationDates;
}
public void setAggregationDates(final List<LocalDate> aggregationDates) {
public void setAggregationDates(final String[] aggregationDates) {
this.aggregationDates = aggregationDates;
}

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java → apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java
View File

@ -4,7 +4,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class FunderUpdate implements Serializable {
public class FunderDbUpdate implements Serializable {
private static final long serialVersionUID = -9086478785780647627L;

11
apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql
View File

@ -5,16 +5,16 @@ CREATE TABLE dsm_service_funder (
_dnet_resource_identifier_ varchar(2048) DEFAULT 'temp_'||md5(clock_timestamp()::text)||'_'||md5(random()::text),
service text NOT NULL REFERENCES dsm_services(id) ON DELETE CASCADE,
funder text NOT NULL REFERENCES dsm_organizations(id) ON DELETE CASCADE,
last_collection_date date,
last_aggregation_date date,
PRIMARY KEY(funder, service)
);
INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_collection_date)
INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_aggregation_date)
SELECT
o.id||'@@'||s.id AS _dnet_resource_identifier_,
s.id AS service,
o.id AS funder,
max(a.last_collection_date::date) AS last_collection_date
max(a.last_aggregation_date::date) AS last_aggregation_date
FROM
dsm_organizations o
JOIN dsm_service_organization so ON (o.id = so.organization)
@ -32,8 +32,9 @@ CREATE OR REPLACE VIEW funders_view AS SELECT
o.country AS country,
o.dateofcollection AS registrationdate,
o.registered_funder AS registered,
array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources,
array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids
array_remove(array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type), NULL) AS datasources,
array_remove(array_agg(DISTINCT sf.last_aggregation_date::text ORDER BY sf.last_aggregation_date::text DESC), NULL) AS aggregationdates,
array_remove(array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid), NULL) AS pids
FROM
dsm_organizations o
JOIN dsm_service_funder sf ON (o.id = sf.funder)