D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'update-funders-apis' (#21) from update-funders-apis into master 

Reviewed-on: #21
This commit is contained in:
Michele Artini 2024-07-15 14:06:51 +02:00
parent 1b9bf9938c b8b190070b
commit 7b855ab0bb
5 changed files with 251 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java
View File

@ -1,97 +1,54 @@
package eu.dnetlib.openaire.funders; package eu.dnetlib.openaire.funders;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate;
import eu.dnetlib.openaire.funders.domain.db.FunderPid; import eu.dnetlib.openaire.funders.domain.db.FunderPid;
@Component @Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public class FunderService { public class FunderService {
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
private static final String SEPARATOR = "@=@"; private static final String SEPARATOR = "@=@";
@Autowired @Autowired
private FunderRepository funderRepository; private FunderRepository funderRepository;
@Autowired @Autowired
private MongoLoggerClient mongoLoggerClient; private JdbcTemplate jdbcTemplate;
private File tempDir; public List<FunderDbEntry> getFunders() {
return StreamSupport.stream(funderRepository.findAll().spliterator(), false)
.map(this::patchFunder)
.collect(Collectors.toList());
private File tempFile;
private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final Log log = LogFactory.getLog(FunderService.class);
@PostConstruct
public void init() {
tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp"));
for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) {
deleteFile(f);
} }
new Thread(this::updateFunders).start(); public FunderDbEntry getFunder(final String id) throws FundersApiException {
return funderRepository.findById(id)
.map(this::patchFunder)
.orElseThrow(() -> new FundersApiException("Missing Funder: " + id));
} }
private void deleteFile(final File f) { public boolean isValidFunder(final String id) {
if (f != null && f.exists()) { return funderRepository.existsById(id);
log.info("Deleting file: " + f.getAbsolutePath());
f.delete();
}
} }
@Scheduled(cron = "${openaire.exporter.funders.cron}") private FunderDbEntry patchFunder(final FunderDbEntry funder) {
public void updateFunders() {
try {
final ObjectMapper mapper = new ObjectMapper();
mapper.registerModule(new JavaTimeModule());
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
log.info("Generating funders file: " + tmp.getAbsolutePath());
try (final FileWriter writer = new FileWriter(tmp)) {
writer.write("[");
boolean first = true;
for (final FunderDbEntry funder : funderRepository.findAll()) {
log.info(" - adding: " + funder.getId());
// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)
final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres()) final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
.filter(Objects::nonNull) .filter(Objects::nonNull)
.map(s -> s.split(SEPARATOR)) .map(s -> s.split(SEPARATOR))
@ -123,61 +80,48 @@ public class FunderService {
funder.setPids(pids); funder.setPids(pids);
// END PATCH return funder;
addAggregationHistory(funder);
if (first) {
first = false;
} else {
writer.write(",");
}
writer.write(mapper.writeValueAsString(funder));
}
writer.write("]");
log.info("Publish funders file: " + tmp.getAbsolutePath());
deleteFile(tempFile);
setTempFile(tmp);
}
} catch (final Throwable e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
} }
private void addAggregationHistory(final FunderDbEntry funder) { public void updateFunder(final String id, final FunderDbUpdate funderUpdate) {
final List<LocalDate> dates = funder.getDatasources() final String sql =
.stream() "UPDATE dsm_organizations SET ("
.map(FunderDatasource::getId) + " legalshortname,"
.map(id -> { + " legalname,"
try { + " websiteurl,"
return mongoLoggerClient.getAggregationHistoryV2(id); + " logourl,"
} catch (final DsmApiException e) { + " country,"
log.error("Error retrieving the aggregation history", e); + " registered_funder"
throw new RuntimeException("Error retrieving the aggregation history", e); + ") = ("
} + " coalesce(?, legalshortname),"
}) + " coalesce(?, legalname),"
.flatMap(List::stream) + " coalesce(?, websiteurl),"
.filter(AggregationInfo::isCompletedSuccessfully) + " coalesce(?, logourl),"
.filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) + " coalesce(?, country),"
.map(AggregationInfo::getDate) + " coalesce(?, registered_funder)"
.distinct() + ") WHERE id = ?";
.map(s -> LocalDate.parse(s, DATEFORMATTER))
.sorted(Comparator.reverseOrder())
.limit(10)
.collect(Collectors.toList());
funder.setAggregationDates(dates); jdbcTemplate.update(sql, funderUpdate.getLegalShortName(), funderUpdate.getLegalName(), funderUpdate.getWebsiteUrl(), funderUpdate
.getLogoUrl(), funderUpdate.getCountry(), funderUpdate.getRegistered(), id);
if (funderUpdate.getPids() != null) {
funderUpdate.getPids().forEach(pid -> {
// TODO: the first update should be deleted after the re-implementation of the pid-tables,
// TODO: the field 'type' should also be moved in the second update
if (jdbcTemplate.queryForObject("SELECT count(*) FROM dsm_identities WHERE issuertype = ? AND pid = ?", Integer.class, pid.getType(), pid
.getValue()) == 0) {
jdbcTemplate.update("INSERT INTO dsm_identities(issuertype, pid) VALUES (?, ?)", pid.getType(), pid.getValue());
} }
public File getTempFile() { if (jdbcTemplate.queryForObject("SELECT count(*) FROM dsm_organizationpids WHERE organization = ? AND pid = ?", Integer.class, id, pid
return tempFile; .getValue()) == 0) {
jdbcTemplate.update("INSERT INTO dsm_organizationpids(organization, pid) VALUES (?, ?)", id, pid.getValue());
}
});
} }
public void setTempFile(final File tempFile) {
this.tempFile = tempFile;
} }
} }

66
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java
View File

@ -1,25 +1,22 @@
package eu.dnetlib.openaire.funders; package eu.dnetlib.openaire.funders;
import java.io.File; import java.util.List;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderDbUpdate;
import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.responses.ApiResponses;
@ -46,29 +43,46 @@ public class FundersApiController extends AbstractExporterController {
@ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error") @ApiResponse(responseCode = "500", description = "unexpected error")
}) })
public void getFunders(final HttpServletResponse res) throws FundersApiException { public List<FunderDbEntry> getFunders() throws FundersApiException {
try {
res.setContentType(MediaType.APPLICATION_JSON_VALUE); return service.getFunders();
} catch (final Throwable e) {
final File file = service.getTempFile(); log.error("Error getting funders", e);
throw e;
if (file == null) { }
log.error("Missing temp file (NULL)");
throw new FundersApiException("Missing temp file (NULL)");
} }
if (!file.exists()) { @RequestMapping(value = "/funders/{id}", produces = {
log.error("Missing temp file " + service.getTempFile()); "application/json"
throw new FundersApiException("Missing temp file " + service.getTempFile()); }, method = RequestMethod.GET)
@Operation(summary = "get a funder by Id", description = "get a funder by Id")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public FunderDbEntry getFunder(@PathVariable final String id) throws FundersApiException {
try {
return service.getFunder(id);
} catch (final Throwable e) {
log.error("Error getting funder: " + id, e);
throw e;
}
} }
try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) { @RequestMapping(value = "/funders/{id}", produces = {
IOUtils.copy(in, out); "application/json"
return; }, method = RequestMethod.POST)
} catch (final Exception e) { @Operation(summary = "update a funder by Id", description = "update a funder by Id")
log.error("Error reading file " + service.getTempFile(), e); @ApiResponses(value = {
throw new FundersApiException("Error reading file " + service.getTempFile(), e); @ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderDbUpdate funderUpdate) throws FundersApiException {
if (service.isValidFunder(id)) {
service.updateFunder(id, funderUpdate);
return service.getFunder(id);
} }
throw new FundersApiException("Invalid funder: " + id);
} }
} }

21
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java
View File

@ -1,7 +1,7 @@
package eu.dnetlib.openaire.funders.domain.db; package eu.dnetlib.openaire.funders.domain.db;
import java.io.Serializable; import java.io.Serializable;
import java.time.LocalDate; import java.sql.Date;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -16,12 +16,14 @@ import org.hibernate.annotations.TypeDef;
import org.hibernate.annotations.TypeDefs; import org.hibernate.annotations.TypeDefs;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.vladmihalcea.hibernate.type.array.DateArrayType;
import com.vladmihalcea.hibernate.type.array.StringArrayType; import com.vladmihalcea.hibernate.type.array.StringArrayType;
@Entity @Entity
@Table(name = "funders_view") @Table(name = "funders_view")
@TypeDefs({ @TypeDefs({
@TypeDef(name = "string-array", typeClass = StringArrayType.class) @TypeDef(name = "string-array", typeClass = StringArrayType.class),
@TypeDef(name = "date-array", typeClass = DateArrayType.class),
}) })
public class FunderDbEntry implements Serializable { public class FunderDbEntry implements Serializable {
@ -47,7 +49,7 @@ public class FunderDbEntry implements Serializable {
private String country; private String country;
@Column(name = "registrationdate") @Column(name = "registrationdate")
private LocalDate registrationDate; private Date registrationDate;
@Column(name = "registered") @Column(name = "registered")
private Boolean registered; private Boolean registered;
@ -68,8 +70,9 @@ public class FunderDbEntry implements Serializable {
@Transient @Transient
private List<FunderDatasource> datasources = new ArrayList<FunderDatasource>(); private List<FunderDatasource> datasources = new ArrayList<FunderDatasource>();
@Transient @Type(type = "date-array")
private List<LocalDate> aggregationDates; @Column(name = "aggregationdates", columnDefinition = "date[]")
private Date[] aggregationDates;
public String getId() { public String getId() {
return id; return id;
@ -119,11 +122,11 @@ public class FunderDbEntry implements Serializable {
this.country = country; this.country = country;
} }
public LocalDate getRegistrationDate() { public Date getRegistrationDate() {
return registrationDate; return registrationDate;
} }
public void setRegistrationDate(final LocalDate registrationDate) { public void setRegistrationDate(final Date registrationDate) {
this.registrationDate = registrationDate; this.registrationDate = registrationDate;
} }
@ -167,11 +170,11 @@ public class FunderDbEntry implements Serializable {
this.datasources = datasources; this.datasources = datasources;
} }
public List<LocalDate> getAggregationDates() { public Date[] getAggregationDates() {
return aggregationDates; return aggregationDates;
} }
public void setAggregationDates(final List<LocalDate> aggregationDates) { public void setAggregationDates(final Date[] aggregationDates) {
this.aggregationDates = aggregationDates; this.aggregationDates = aggregationDates;
} }

74
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbUpdate.java Normal file
View File

@ -0,0 +1,74 @@
package eu.dnetlib.openaire.funders.domain.db;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class FunderDbUpdate implements Serializable {
private static final long serialVersionUID = -9086478785780647627L;
private String legalShortName;
private String legalName;
private String websiteUrl;
private String logoUrl;
private String country;
private Boolean registered;
private List<FunderPid> pids = new ArrayList<FunderPid>();
public String getLegalShortName() {
return legalShortName;
}
public void setLegalShortName(final String legalShortName) {
this.legalShortName = legalShortName;
}
public String getLegalName() {
return legalName;
}
public void setLegalName(final String legalName) {
this.legalName = legalName;
}
public String getWebsiteUrl() {
return websiteUrl;
}
public void setWebsiteUrl(final String websiteUrl) {
this.websiteUrl = websiteUrl;
}
public String getLogoUrl() {
return logoUrl;
}
public void setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
public Boolean getRegistered() {
return registered;
}
public void setRegistered(final Boolean registered) {
this.registered = registered;
}
public List<FunderPid> getPids() {
return pids;
}
public void setPids(final List<FunderPid> pids) {
this.pids = pids;
}
}

38
apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql
View File

@ -1,6 +1,29 @@
ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean; ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean;
CREATE VIEW funders_view AS SELECT
CREATE TABLE dsm_service_funder (
_dnet_resource_identifier_ varchar(2048) DEFAULT 'temp_'||md5(clock_timestamp()::text)||'_'||md5(random()::text),
service text NOT NULL REFERENCES dsm_services(id) ON DELETE CASCADE,
funder text NOT NULL REFERENCES dsm_organizations(id) ON DELETE CASCADE,
last_aggregation_date date,
PRIMARY KEY(funder, service)
);
INSERT INTO dsm_service_funder(_dnet_resource_identifier_, service, funder, last_aggregation_date)
SELECT
o.id||'@@'||s.id AS _dnet_resource_identifier_,
s.id AS service,
o.id AS funder,
max(a.last_aggregation_date::date) AS last_aggregation_date
FROM
dsm_organizations o
JOIN dsm_service_organization so ON (o.id = so.organization)
JOIN dsm_services s ON (so.service = s.id)
JOIN projects p ON p.collectedfrom = s.id
LEFT OUTER JOIN dsm_api a ON (s.id = a.service)
GROUP BY s.id, o.id;
CREATE OR REPLACE VIEW funders_view AS SELECT
o.id AS id, o.id AS id,
o.legalshortname AS legalshortname, o.legalshortname AS legalshortname,
o.legalname AS legalname, o.legalname AS legalname,
@ -9,17 +32,16 @@ CREATE VIEW funders_view AS SELECT
o.country AS country, o.country AS country,
o.dateofcollection AS registrationdate, o.dateofcollection AS registrationdate,
o.registered_funder AS registered, o.registered_funder AS registered,
array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources, array_remove(array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type), NULL) AS datasources,
array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids array_remove(array_agg(DISTINCT sf.last_aggregation_date ORDER BY sf.last_aggregation_date DESC), NULL) AS aggregationdates,
array_remove(array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid), NULL) AS pids
FROM FROM
dsm_organizations o dsm_organizations o
JOIN dsm_service_organization so ON (o.id = so.organization) JOIN dsm_service_funder sf ON (o.id = sf.funder)
JOIN dsm_services s ON (so.service = s.id) JOIN dsm_services s ON (sf.service = s.id)
JOIN projects p ON (p.collectedfrom = s.id)
LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization) LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization)
LEFT OUTER JOIN dsm_identities pids ON (opids.pid = pids.pid) LEFT OUTER JOIN dsm_identities pids ON (opids.pid = pids.pid)
GROUP BY o.id; GROUP BY o.id;
GRANT ALL ON dsm_service_funder TO dnetapi;
GRANT ALL ON funders_view TO dnetapi; GRANT ALL ON funders_view TO dnetapi;