diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index 04ac9d4b..8f7200cb 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -113,6 +113,10 @@ com.fasterxml.jackson.datatype jackson-datatype-joda + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + joda-time joda-time diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java new file mode 100644 index 00000000..34a6cc49 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java @@ -0,0 +1,22 @@ +package eu.dnetlib.openaire.common; + +import java.util.Optional; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.NoRepositoryBean; +import org.springframework.data.repository.Repository; + +@NoRepositoryBean +public interface ReadOnlyRepository extends Repository { + + Optional findById(ID id); + + boolean existsById(ID id); + + Page findAll(Pageable pageable); + + Iterable findAll(); + + long count(); +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java deleted file mode 100644 index be41be45..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java +++ /dev/null @@ -1,91 +0,0 @@ -package eu.dnetlib.openaire.funders; - -import java.io.StringReader; -import java.util.stream.Collectors; - -import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FundingStream; -import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; -import eu.dnetlib.openaire.funders.domain.db.FundingPathDbEntry; -import org.dom4j.Document; -import org.dom4j.DocumentException; -import org.dom4j.io.SAXReader; - -import static org.apache.commons.lang3.StringUtils.*; - -public class ConversionUtils { - - public static final String SEPARATOR = "::"; - - public static FunderDetails asFunderDetails(final FunderDbEntry fdb) { - final FunderDetails f = new FunderDetails(); - - f.setId(fdb.getId()); - f.setName(fdb.getName()); - f.setShortname(fdb.getShortname()); - f.setJurisdiction(fdb.getJurisdiction()); - f.setRegistrationDate(fdb.getRegistrationdate()); - f.setLastUpdateDate(fdb.getLastupdatedate()); - - return f; - } - - public static ExtendedFunderDetails asExtendedFunderDetails(final FunderDbEntry fdb) { - final ExtendedFunderDetails f = new ExtendedFunderDetails(asFunderDetails(fdb)); - - if (fdb.getFundingpaths() != null) { - f.setFundingStreams( - fdb.getFundingpaths().stream() - .map(ConversionUtils::asFundingStream) - .collect(Collectors.toList())); - } - - return f; - } - - private static FundingStream asFundingStream(final FundingPathDbEntry pathDbEntry) { - final FundingStream f = new FundingStream(); - - try { - final Document xml = new SAXReader().read(new StringReader(pathDbEntry.getPath())); - - for(int i=2;i>=0;i--) { - if (hasFundingLevel(i, xml)) { - f.setId(getId(i, xml)); - f.setName(getName(i, xml)); - break; - } - } - - if (isBlank(f.getId()) && isNoneBlank(xml.valueOf("//funder/id/text()"))) { - f.setId(xml.valueOf("//funder/shortname/text()")); - f.setName(xml.valueOf("//funder/name/text()")); - } - - if (isBlank(f.getId())) { - throw new IllegalStateException("invalid funding path:\n" + xml.asXML()); - } - - return f; - - } catch (DocumentException e) { - throw new IllegalStateException("unable to parse funding path:\n" + pathDbEntry.getPath()); - } - } - - private static String getName(int level, final Document d) { - return d.valueOf(String.format("//funding_level_%s/description/text()", level)); - } - - private static String getId(int level, final Document d) { - return substringAfter( - d.valueOf( - String.format("//funding_level_%s/id/text()", level) - ), SEPARATOR); - } - - private static boolean hasFundingLevel(int level, Document d) { - return isNotBlank(getId(level, d)); - } -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java deleted file mode 100644 index 9c79afc5..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java +++ /dev/null @@ -1,43 +0,0 @@ -package eu.dnetlib.openaire.funders; - -import java.util.List; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.data.domain.PageRequest; -import org.springframework.stereotype.Component; - -import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; - -@Component -@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") -public class FunderDao { - - @Autowired - private FunderRepository funderRepository; - - public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException { - return ConversionUtils - .asExtendedFunderDetails(funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId))); - } - - public List listFunderDetails(final int page, final int size) throws FundersApiException { - return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(ConversionUtils::asFunderDetails) - .collect(Collectors.toList()); - } - - public List listFunderIds(final int page, final int size) throws FundersApiException { - return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(f -> f.getId()) - .collect(Collectors.toList()); - } - -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java index 39bec085..57aeebc3 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java @@ -1,10 +1,11 @@ package eu.dnetlib.openaire.funders; -import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import eu.dnetlib.openaire.common.ReadOnlyRepository; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; + @Repository -public interface FunderRepository extends JpaRepository { +public interface FunderRepository extends ReadOnlyRepository { } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java new file mode 100644 index 00000000..756259c9 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -0,0 +1,183 @@ +package eu.dnetlib.openaire.funders; + +import java.io.File; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; +import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; +import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; +import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; +import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FunderPid; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") +public class FunderService { + + private static final String TEMP_FILE_SUFFIX = ".funds.tmp"; + private static final String SEPARATOR = "@=@"; + + @Autowired + private FunderRepository funderRepository; + + @Autowired + private MongoLoggerClient mongoLoggerClient; + + private File tempDir; + + private File tempFile; + + private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + private static final Log log = LogFactory.getLog(FunderService.class); + + @PostConstruct + public void init() { + + tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp")); + + for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) { + deleteFile(f); + } + + new Thread(this::updateFunders).start(); + } + + private void deleteFile(final File f) { + if (f != null && f.exists()) { + log.info("Deleting file: " + f.getAbsolutePath()); + f.delete(); + } + } + + @Scheduled(cron = "${openaire.exporter.funders.cron}") + public void updateFunders() { + try { + final ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir); + + log.info("Generating funders file: " + tmp.getAbsolutePath()); + + try (final FileWriter writer = new FileWriter(tmp)) { + writer.write("["); + boolean first = true; + for (final FunderDbEntry funder : funderRepository.findAll()) { + log.info(" - adding: " + funder.getId()); + + // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) + + final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 3) + .map(arr -> { + final FunderDatasource ds = new FunderDatasource(); + ds.setId(arr[0].trim()); + ds.setName(arr[1].trim()); + ds.setType(arr[2].trim()); + return ds; + }) + .filter(ds -> StringUtils.isNotBlank(ds.getId())) + .collect(Collectors.toList()); + + funder.setDatasources(datasources); + + final List pids = Arrays.stream(funder.getPidsPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 2) + .map(arr -> { + final FunderPid pid = new FunderPid(); + pid.setType(arr[0].trim()); + pid.setValue(arr[1].trim()); + return pid; + }) + .filter(pid -> StringUtils.isNotBlank(pid.getValue())) + .collect(Collectors.toList()); + + funder.setPids(pids); + + // END PATCH + + addAggregationHistory(funder); + + if (first) { + first = false; + } else { + writer.write(","); + } + writer.write(mapper.writeValueAsString(funder)); + } + writer.write("]"); + log.info("Publish funders file: " + tmp.getAbsolutePath()); + + deleteFile(tempFile); + setTempFile(tmp); + } + } catch (final Throwable e) { + log.error("Error generating funders file", e); + throw new RuntimeException("Error generating funders file", e); + } + } + + private void addAggregationHistory(final FunderDbEntry funder) { + + final List dates = funder.getDatasources() + .stream() + .map(FunderDatasource::getId) + .map(id -> { + try { + return mongoLoggerClient.getAggregationHistoryV2(id); + } catch (final DsmApiException e) { + log.error("Error retrieving the aggregation history", e); + throw new RuntimeException("Error retrieving the aggregation history", e); + } + }) + .flatMap(List::stream) + .filter(AggregationInfo::isCompletedSuccessfully) + .filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) + .map(AggregationInfo::getDate) + .distinct() + .map(s -> LocalDate.parse(s, DATEFORMATTER)) + .sorted(Comparator.reverseOrder()) + .limit(10) + .collect(Collectors.toList()); + + funder.setAggregationDates(dates); + } + + public File getTempFile() { + return tempFile; + } + + public void setTempFile(final File tempFile) { + this.tempFile = tempFile; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index d1533826..f4b0bf8d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -1,21 +1,25 @@ package eu.dnetlib.openaire.funders; -import java.util.List; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -23,59 +27,48 @@ import io.swagger.v3.oas.annotations.tags.Tag; @RestController @CrossOrigin(origins = { - "*" + "*" }) @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") @Tag(name = "OpenAIRE funders API", description = "the OpenAIRE funders API") public class FundersApiController extends AbstractExporterController { + @Autowired + private FunderService service; + private static final Log log = LogFactory.getLog(FundersApiController.class); - @Autowired - private FunderDao fDao; - @RequestMapping(value = "/funders", produces = { - "application/json" + "application/json" }, method = RequestMethod.GET) - @Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, last update date, registration date") + @Operation(summary = "get all funders", description = "get all funders") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") }) - public List getFunders( - @PathVariable final int page, - @PathVariable final int size) throws FundersApiException { + public void getFunders(final HttpServletResponse res) throws FundersApiException { - return fDao.listFunderDetails(page, size); - } + res.setContentType(MediaType.APPLICATION_JSON_VALUE); - @RequestMapping(value = "/funder/{id}", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "get the funder details", description = "complete funder information") - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public ExtendedFunderDetails getFunderDetails( - @PathVariable final String id) throws FundersApiException { + final File file = service.getTempFile(); - return fDao.getExtendedFunderDetails(id); - } + if (file == null) { + log.error("Missing temp file (NULL)"); + throw new FundersApiException("Missing temp file (NULL)"); + } - @RequestMapping(value = "/funder/ids", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "get the list of funder ids", description = "get the list of funder ids") - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public List getFunderIds( - @PathVariable final int page, - @PathVariable final int size) throws FundersApiException { + if (!file.exists()) { + log.error("Missing temp file " + service.getTempFile()); + throw new FundersApiException("Missing temp file " + service.getTempFile()); + } - return fDao.listFunderIds(page, size); + try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) { + IOUtils.copy(in, out); + return; + } catch (final Exception e) { + log.error("Error reading file " + service.getTempFile(), e); + throw new FundersApiException("Error reading file " + service.getTempFile(), e); + } } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java new file mode 100644 index 00000000..96556de0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java @@ -0,0 +1,39 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.io.Serializable; + +public class FunderDatasource implements Serializable { + + private static final long serialVersionUID = 2145493560459874509L; + + private String id; + + private String name; + + private String type; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index a47c8d51..1a3219c8 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -1,38 +1,75 @@ package eu.dnetlib.openaire.funders.domain.db; -import java.sql.Date; -import java.util.Set; -import javax.persistence.*; +import java.io.Serializable; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; -import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Transient; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.vladmihalcea.hibernate.type.array.StringArrayType; @Entity -@Table(name = "funders") -public class FunderDbEntry { +@Table(name = "funders_view") +@TypeDefs({ + @TypeDef(name = "string-array", typeClass = StringArrayType.class) +}) +public class FunderDbEntry implements Serializable { + + private static final long serialVersionUID = 1290088460508203016L; @Id + @Column(name = "id") private String id; - private String name; - private String shortname; - private String jurisdiction; - private String websiteurl; - private String policy; - private Date registrationdate; - private Date lastupdatedate; - @OneToMany( - cascade = { CascadeType.PERSIST, CascadeType.MERGE }, - fetch = FetchType.LAZY ) - @JoinTable( - name = "funder_identity", - joinColumns = @JoinColumn(name = "funder"), - inverseJoinColumns = @JoinColumn(name = "pid")) - private Set pids; + @Column(name = "legalshortname") + private String legalShortName; - @OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY) - private Set fundingpaths; + @Column(name = "legalname") + private String legalName; - public FunderDbEntry() {} + @Column(name = "websiteurl") + private String websiteUrl; + + @Column(name = "logourl") + private String logoUrl; + + @Column(name = "country") + private String country; + + @Column(name = "registrationdate") + private LocalDate registrationDate; + + @Column(name = "registered") + private Boolean registered; + + @JsonIgnore + @Type(type = "string-array") + @Column(name = "pids", columnDefinition = "text[]") + private String[] pidsPostgres; + + @Transient + private List pids = new ArrayList(); + + @JsonIgnore + @Type(type = "string-array") + @Column(name = "datasources", columnDefinition = "text[]") + private String[] datasourcesPostgres; + + @Transient + private List datasources = new ArrayList(); + + @Transient + private List aggregationDates; public String getId() { return id; @@ -42,75 +79,100 @@ public class FunderDbEntry { this.id = id; } - public String getName() { - return name; + public String getLegalShortName() { + return legalShortName; } - public void setName(final String name) { - this.name = name; + public void setLegalShortName(final String legalShortName) { + this.legalShortName = legalShortName; } - public String getShortname() { - return shortname; + public String getLegalName() { + return legalName; } - public void setShortname(final String shortname) { - this.shortname = shortname; + public void setLegalName(final String legalName) { + this.legalName = legalName; } - public String getJurisdiction() { - return jurisdiction; + public String getWebsiteUrl() { + return websiteUrl; } - public void setJurisdiction(final String jurisdiction) { - this.jurisdiction = jurisdiction; + public void setWebsiteUrl(final String websiteUrl) { + this.websiteUrl = websiteUrl; } - public String getWebsiteurl() { - return websiteurl; + public String getLogoUrl() { + return logoUrl; } - public void setWebsiteurl(final String websiteurl) { - this.websiteurl = websiteurl; + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; } - public String getPolicy() { - return policy; + public String getCountry() { + return country; } - public void setPolicy(final String policy) { - this.policy = policy; + public void setCountry(final String country) { + this.country = country; } - public Date getRegistrationdate() { - return registrationdate; + public LocalDate getRegistrationDate() { + return registrationDate; } - public void setRegistrationdate(final Date registrationdate) { - this.registrationdate = registrationdate; + public void setRegistrationDate(final LocalDate registrationDate) { + this.registrationDate = registrationDate; } - public Date getLastupdatedate() { - return lastupdatedate; + public Boolean getRegistered() { + return registered; } - public void setLastupdatedate(final Date lastupdatedate) { - this.lastupdatedate = lastupdatedate; + public void setRegistered(final Boolean registered) { + this.registered = registered; } - public Set getPids() { + public String[] getPidsPostgres() { + return pidsPostgres; + } + + public void setPidsPostgres(final String[] pidsPostgres) { + this.pidsPostgres = pidsPostgres; + } + + public List getPids() { return pids; } - public void setPids(final Set pids) { + public void setPids(final List pids) { this.pids = pids; } - public Set getFundingpaths() { - return fundingpaths; + public String[] getDatasourcesPostgres() { + return datasourcesPostgres; } - public void setFundingpaths(final Set fundingpaths) { - this.fundingpaths = fundingpaths; + public void setDatasourcesPostgres(final String[] datasourcesPostgres) { + this.datasourcesPostgres = datasourcesPostgres; } + + public List getDatasources() { + return datasources; + } + + public void setDatasources(final List datasources) { + this.datasources = datasources; + } + + public List getAggregationDates() { + return aggregationDates; + } + + public void setAggregationDates(final List aggregationDates) { + this.aggregationDates = aggregationDates; + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java new file mode 100644 index 00000000..bf058cb4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.io.Serializable; + +public class FunderPid implements Serializable { + + private static final long serialVersionUID = 2145493560459874509L; + + private String type; + + private String value; + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java new file mode 100644 index 00000000..2a79306b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java @@ -0,0 +1,118 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.sql.Date; +import java.util.Set; + +import javax.persistence.CascadeType; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.OneToMany; + +import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; + +// @Entity +// @Table(name = "funders") +@Deprecated +public class OldFunderDbEntry { + + @Id + private String id; + private String name; + private String shortname; + private String jurisdiction; + private String websiteurl; + private String policy; + private Date registrationdate; + private Date lastupdatedate; + + @OneToMany(cascade = { CascadeType.PERSIST, CascadeType.MERGE }, fetch = FetchType.LAZY) + @JoinTable(name = "funder_identity", joinColumns = @JoinColumn(name = "funder"), inverseJoinColumns = @JoinColumn(name = "pid")) + private Set pids; + + @OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + private Set fundingpaths; + + public OldFunderDbEntry() {} + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortname() { + return shortname; + } + + public void setShortname(final String shortname) { + this.shortname = shortname; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(final String policy) { + this.policy = policy; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + + public Date getLastupdatedate() { + return lastupdatedate; + } + + public void setLastupdatedate(final Date lastupdatedate) { + this.lastupdatedate = lastupdatedate; + } + + public Set getPids() { + return pids; + } + + public void setPids(final Set pids) { + this.pids = pids; + } + + public Set getFundingpaths() { + return fundingpaths; + } + + public void setFundingpaths(final Set fundingpaths) { + this.fundingpaths = fundingpaths; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java similarity index 88% rename from apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java rename to apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java index 2f7dc534..d5711b66 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java @@ -1,12 +1,15 @@ package eu.dnetlib.openaire.funders.domain.db; -import javax.persistence.*; +import javax.persistence.Column; +import javax.persistence.Id; +import javax.persistence.Transient; import com.fasterxml.jackson.annotation.JsonIgnore; -@Entity -@Table(name = "fundingpaths") -public class FundingPathDbEntry { +// @Entity +// @Table(name = "fundingpaths") +@Deprecated +public class OldFundingPathDbEntry { @JsonIgnore @Transient @@ -28,7 +31,7 @@ public class FundingPathDbEntry { private String funderid; - public FundingPathDbEntry() {} + public OldFundingPathDbEntry() {} public String getDnetresourceidentifier() { return dnetresourceidentifier; diff --git a/apps/dnet-exporter-api/src/main/resources/application.properties b/apps/dnet-exporter-api/src/main/resources/application.properties index 3aeb5b58..19150128 100644 --- a/apps/dnet-exporter-api/src/main/resources/application.properties +++ b/apps/dnet-exporter-api/src/main/resources/application.properties @@ -27,7 +27,7 @@ openaire.exporter.enable.dsm = true openaire.exporter.enable.community = true openaire.exporter.enable.community.import = false openaire.exporter.enable.context = true -openaire.exporter.enable.funders = false +openaire.exporter.enable.funders = true openaire.exporter.enable.project = true openaire.exporter.enable.info = true @@ -36,3 +36,4 @@ openaire.exporter.cache.ttl = 43200000 maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml +openaire.exporter.funders.cron = 0 30 2 * * * diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql new file mode 100644 index 00000000..9e733f32 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -0,0 +1,25 @@ +ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean; + +CREATE VIEW funders_view AS SELECT + o.id AS id, + o.legalshortname AS legalshortname, + o.legalname AS legalname, + o.websiteurl AS websiteurl, + o.logourl AS logourl, + o.country AS country, + o.dateofcollection AS registrationdate, + o.registered_funder AS registered, + array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources, + array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids +FROM + dsm_organizations o + JOIN dsm_service_organization so ON (o.id = so.organization) + JOIN dsm_services s ON (so.service = s.id) + JOIN projects p ON (p.collectedfrom = s.id) + LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization) + LEFT OUTER JOIN dsm_identities pids ON (opids.pid = pids.pid) +GROUP BY o.id; + +GRANT ALL ON funders_view TO dnetapi; + + diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java index 8869dba4..b2fef0e1 100644 --- a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java @@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; public class FunderContextClientTest { - private FunderDao fDao; + private FunderService fDao; @BeforeEach public void setUp() { - fDao = new FunderDao(); + fDao = new FunderService(); } @Test diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java index 723feafb..51e0ecff 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java @@ -4,6 +4,7 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonAutoDetect; +@Deprecated @JsonAutoDetect public class ExtendedFunderDetails extends FunderDetails { diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java index 863a443b..0f234854 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java @@ -1,5 +1,6 @@ package eu.dnetlib.openaire.exporter.model.funders; +@Deprecated public class FundingStream { private String id;