D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new api ti update funders

This commit is contained in:
Michele Artini 2024-06-25 11:44:53 +02:00
parent 3a5402d0af
commit e060a3601c
3 changed files with 207 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.openaire.funders;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
@ -10,6 +11,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.annotation.PostConstruct;
@ -18,20 +20,24 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
import eu.dnetlib.openaire.exporter.exceptions.DsmApiException;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderPid;
import eu.dnetlib.openaire.funders.domain.db.FunderUpdate;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
@ -43,6 +49,9 @@ public class FunderService {
@Autowired
private FunderRepository funderRepository;
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
private MongoLoggerClient mongoLoggerClient;
@ -75,23 +84,51 @@ public class FunderService {
@Scheduled(cron = "${openaire.exporter.funders.cron}")
public void updateFunders() {
try {
final ObjectMapper mapper = new ObjectMapper();
mapper.registerModule(new JavaTimeModule());
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
final String records = StreamSupport.stream(funderRepository.findAll().spliterator(), false)
.map(this::patchFunder)
.map(this::addAggregationHistory)
.map(funder -> {
try {
return mapper.writeValueAsString(funder);
} catch (final JsonProcessingException e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
})
.collect(Collectors.joining(","));
try {
final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir);
log.info("Generating funders file: " + tmp.getAbsolutePath());
try (final FileWriter writer = new FileWriter(tmp)) {
writer.write("[");
boolean first = true;
for (final FunderDbEntry funder : funderRepository.findAll()) {
log.info(" - adding: " + funder.getId());
writer.write(records);
writer.write("]");
log.info("Publish funders file: " + tmp.getAbsolutePath());
deleteFile(tempFile);
setTempFile(tmp);
}
} catch (final IOException e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
}
public FunderDbEntry getFunder(final String id) throws FundersApiException {
return funderRepository.findById(id)
.map(this::patchFunder)
.map(this::addAggregationHistory)
.orElseThrow(() -> new FundersApiException("Missing Funder: " + id));
}
private FunderDbEntry patchFunder(final FunderDbEntry funder) {
// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)
final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
.filter(Objects::nonNull)
.map(s -> s.split(SEPARATOR))
@ -123,30 +160,10 @@ public class FunderService {
funder.setPids(pids);
// END PATCH
addAggregationHistory(funder);
if (first) {
first = false;
} else {
writer.write(",");
}
writer.write(mapper.writeValueAsString(funder));
}
writer.write("]");
log.info("Publish funders file: " + tmp.getAbsolutePath());
deleteFile(tempFile);
setTempFile(tmp);
}
} catch (final Throwable e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}
return funder;
}
private void addAggregationHistory(final FunderDbEntry funder) {
private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) {
final List<LocalDate> dates = funder.getDatasources()
.stream()
@ -170,6 +187,49 @@ public class FunderService {
.collect(Collectors.toList());
funder.setAggregationDates(dates);
return funder;
}
public void updateFunder(final String id, final FunderUpdate funderUpdate) {
final String sql =
"UPDATE dsm_organizations SET ("
+ " legalshortname,"
+ " legalname,"
+ " websiteurl,"
+ " logourl,"
+ " country,"
+ " registered_funder"
+ ") = ("
+ " coalesce(?, legalshortname),"
+ " coalesce(?, legalname),"
+ " coalesce(?, websiteurl),"
+ " coalesce(?, logourl),"
+ " coalesce(?, country),"
+ " coalesce(?, registered_funder)"
+ ") WHERE id = ?";
jdbcTemplate.update(sql, funderUpdate.getLegalShortName(), funderUpdate.getLegalName(), funderUpdate.getWebsiteUrl(), funderUpdate
.getLogoUrl(), funderUpdate.getCountry(), funderUpdate.getRegistered(), id);
if (funderUpdate.getPids() != null) {
funderUpdate.getPids().forEach(pid -> {
// TODO: the first update should be deleted after the re-implementation of the pid-tables,
// TODO: the field 'type' should also be moved in the second update
if (jdbcTemplate.queryForObject("SELECT count(*) FROM dsm_identities WHERE issuertype = ? AND pid = ?", Integer.class, pid.getType(), pid
.getValue()) == 0) {
jdbcTemplate.update("INSERT INTO dsm_identities(issuertype, pid) VALUES (?, ?)", pid.getType(), pid.getValue());
}
if (jdbcTemplate.queryForObject("SELECT count(*) FROM dsm_organizationpids WHERE organization = ? AND pid = ?", Integer.class, id, pid
.getValue()) == 0) {
jdbcTemplate.update("INSERT INTO dsm_organizationpids(organization, pid) VALUES (?, ?)", id, pid.getValue());
}
});
}
}
public File getTempFile() {

18
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java
View File

@ -14,12 +14,17 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.openaire.common.AbstractExporterController;
import eu.dnetlib.openaire.exporter.exceptions.FundersApiException;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderUpdate;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
@ -71,4 +76,17 @@ public class FundersApiController extends AbstractExporterController {
}
}
@RequestMapping(value = "/funders/{id}", produces = {
"application/json"
}, method = RequestMethod.POST)
@Operation(summary = "update a funder by Id", description = "update a funder by Id")
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "OK"),
@ApiResponse(responseCode = "500", description = "unexpected error")
})
public @ResponseBody FunderDbEntry updateFunder(@PathVariable final String id, @RequestBody final FunderUpdate funderUpdate) throws FundersApiException {
service.updateFunder(id, funderUpdate);
return service.getFunder(id);
}
}

74
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderUpdate.java Normal file
View File

@ -0,0 +1,74 @@
package eu.dnetlib.openaire.funders.domain.db;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class FunderUpdate implements Serializable {
private static final long serialVersionUID = -9086478785780647627L;
private String legalShortName;
private String legalName;
private String websiteUrl;
private String logoUrl;
private String country;
private Boolean registered;
private List<FunderPid> pids = new ArrayList<FunderPid>();
public String getLegalShortName() {
return legalShortName;
}
public void setLegalShortName(final String legalShortName) {
this.legalShortName = legalShortName;
}
public String getLegalName() {
return legalName;
}
public void setLegalName(final String legalName) {
this.legalName = legalName;
}
public String getWebsiteUrl() {
return websiteUrl;
}
public void setWebsiteUrl(final String websiteUrl) {
this.websiteUrl = websiteUrl;
}
public String getLogoUrl() {
return logoUrl;
}
public void setLogoUrl(final String logoUrl) {
this.logoUrl = logoUrl;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
public Boolean getRegistered() {
return registered;
}
public void setRegistered(final Boolean registered) {
this.registered = registered;
}
public List<FunderPid> getPids() {
return pids;
}
public void setPids(final List<FunderPid> pids) {
this.pids = pids;
}
}