From e5901a0181a906f99276369d5a428f86470a50b5 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 29 Jan 2024 15:01:03 +0100 Subject: [PATCH 1/7] new funder apis --- .../openaire/common/ReadOnlyRepository.java | 22 ++++ .../openaire/funders/ConversionUtils.java | 91 ------------- .../dnetlib/openaire/funders/FunderDao.java | 25 ++-- .../openaire/funders/FunderRepository.java | 7 +- .../funders/FundersApiController.java | 49 +++---- .../funders/domain/db/FunderDbEntry.java | 122 ++++++++---------- .../funders/domain/db/OldFunderDbEntry.java | 118 +++++++++++++++++ ...bEntry.java => OldFundingPathDbEntry.java} | 13 +- .../src/main/resources/sql/funders-schema.sql | 17 +++ .../model/funders/ExtendedFunderDetails.java | 1 + .../exporter/model/funders/FundingStream.java | 1 + 11 files changed, 259 insertions(+), 207 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java delete mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java rename apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/{FundingPathDbEntry.java => OldFundingPathDbEntry.java} (88%) create mode 100644 apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java new file mode 100644 index 00000000..34a6cc49 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ReadOnlyRepository.java @@ -0,0 +1,22 @@ +package eu.dnetlib.openaire.common; + +import java.util.Optional; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.NoRepositoryBean; +import org.springframework.data.repository.Repository; + +@NoRepositoryBean +public interface ReadOnlyRepository extends Repository { + + Optional findById(ID id); + + boolean existsById(ID id); + + Page findAll(Pageable pageable); + + Iterable findAll(); + + long count(); +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java deleted file mode 100644 index be41be45..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/ConversionUtils.java +++ /dev/null @@ -1,91 +0,0 @@ -package eu.dnetlib.openaire.funders; - -import java.io.StringReader; -import java.util.stream.Collectors; - -import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FundingStream; -import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; -import eu.dnetlib.openaire.funders.domain.db.FundingPathDbEntry; -import org.dom4j.Document; -import org.dom4j.DocumentException; -import org.dom4j.io.SAXReader; - -import static org.apache.commons.lang3.StringUtils.*; - -public class ConversionUtils { - - public static final String SEPARATOR = "::"; - - public static FunderDetails asFunderDetails(final FunderDbEntry fdb) { - final FunderDetails f = new FunderDetails(); - - f.setId(fdb.getId()); - f.setName(fdb.getName()); - f.setShortname(fdb.getShortname()); - f.setJurisdiction(fdb.getJurisdiction()); - f.setRegistrationDate(fdb.getRegistrationdate()); - f.setLastUpdateDate(fdb.getLastupdatedate()); - - return f; - } - - public static ExtendedFunderDetails asExtendedFunderDetails(final FunderDbEntry fdb) { - final ExtendedFunderDetails f = new ExtendedFunderDetails(asFunderDetails(fdb)); - - if (fdb.getFundingpaths() != null) { - f.setFundingStreams( - fdb.getFundingpaths().stream() - .map(ConversionUtils::asFundingStream) - .collect(Collectors.toList())); - } - - return f; - } - - private static FundingStream asFundingStream(final FundingPathDbEntry pathDbEntry) { - final FundingStream f = new FundingStream(); - - try { - final Document xml = new SAXReader().read(new StringReader(pathDbEntry.getPath())); - - for(int i=2;i>=0;i--) { - if (hasFundingLevel(i, xml)) { - f.setId(getId(i, xml)); - f.setName(getName(i, xml)); - break; - } - } - - if (isBlank(f.getId()) && isNoneBlank(xml.valueOf("//funder/id/text()"))) { - f.setId(xml.valueOf("//funder/shortname/text()")); - f.setName(xml.valueOf("//funder/name/text()")); - } - - if (isBlank(f.getId())) { - throw new IllegalStateException("invalid funding path:\n" + xml.asXML()); - } - - return f; - - } catch (DocumentException e) { - throw new IllegalStateException("unable to parse funding path:\n" + pathDbEntry.getPath()); - } - } - - private static String getName(int level, final Document d) { - return d.valueOf(String.format("//funding_level_%s/description/text()", level)); - } - - private static String getId(int level, final Document d) { - return substringAfter( - d.valueOf( - String.format("//funding_level_%s/id/text()", level) - ), SEPARATOR); - } - - private static boolean hasFundingLevel(int level, Document d) { - return isNotBlank(getId(level, d)); - } -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java index 9c79afc5..5cde4c94 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java @@ -9,8 +9,7 @@ import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") @@ -19,25 +18,23 @@ public class FunderDao { @Autowired private FunderRepository funderRepository; - public ExtendedFunderDetails getExtendedFunderDetails(final String funderId) throws FundersApiException { - return ConversionUtils - .asExtendedFunderDetails(funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId))); + public FunderDbEntry findFunder(final String funderId) throws FundersApiException { + return funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId)); } - public List listFunderDetails(final int page, final int size) throws FundersApiException { + public List listFunders(final int page, final int size) throws FundersApiException { return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(ConversionUtils::asFunderDetails) - .collect(Collectors.toList()); + .getContent() + .stream() + .collect(Collectors.toList()); } public List listFunderIds(final int page, final int size) throws FundersApiException { return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(f -> f.getId()) - .collect(Collectors.toList()); + .getContent() + .stream() + .map(FunderDbEntry::getId) + .collect(Collectors.toList()); } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java index 39bec085..57aeebc3 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderRepository.java @@ -1,10 +1,11 @@ package eu.dnetlib.openaire.funders; -import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import eu.dnetlib.openaire.common.ReadOnlyRepository; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; + @Repository -public interface FunderRepository extends JpaRepository { +public interface FunderRepository extends ReadOnlyRepository { } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index d1533826..e1e16177 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -2,8 +2,6 @@ package eu.dnetlib.openaire.funders; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.web.bind.annotation.CrossOrigin; @@ -14,8 +12,7 @@ import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.funders.ExtendedFunderDetails; -import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -23,57 +20,53 @@ import io.swagger.v3.oas.annotations.tags.Tag; @RestController @CrossOrigin(origins = { - "*" + "*" }) @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") @Tag(name = "OpenAIRE funders API", description = "the OpenAIRE funders API") public class FundersApiController extends AbstractExporterController { - private static final Log log = LogFactory.getLog(FundersApiController.class); - @Autowired private FunderDao fDao; @RequestMapping(value = "/funders", produces = { - "application/json" + "application/json" }, method = RequestMethod.GET) - @Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, last update date, registration date") + @Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, registration date") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") }) - public List getFunders( - @PathVariable final int page, - @PathVariable final int size) throws FundersApiException { + public List getFunders( + @PathVariable final int page, + @PathVariable final int size) throws FundersApiException { - return fDao.listFunderDetails(page, size); + return fDao.listFunders(page, size); } @RequestMapping(value = "/funder/{id}", produces = { - "application/json" + "application/json" }, method = RequestMethod.GET) @Operation(summary = "get the funder details", description = "complete funder information") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") }) - public ExtendedFunderDetails getFunderDetails( - @PathVariable final String id) throws FundersApiException { - - return fDao.getExtendedFunderDetails(id); + public FunderDbEntry getFunderDetails(@PathVariable final String id) throws FundersApiException { + return fDao.findFunder(id); } @RequestMapping(value = "/funder/ids", produces = { - "application/json" + "application/json" }, method = RequestMethod.GET) @Operation(summary = "get the list of funder ids", description = "get the list of funder ids") @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") }) - public List getFunderIds( - @PathVariable final int page, - @PathVariable final int size) throws FundersApiException { + public List listFunderIds( + @PathVariable final int page, + @PathVariable final int size) throws FundersApiException { return fDao.listFunderIds(page, size); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index a47c8d51..0d66ed8f 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -1,38 +1,43 @@ package eu.dnetlib.openaire.funders.domain.db; -import java.sql.Date; -import java.util.Set; -import javax.persistence.*; +import java.io.Serializable; +import java.time.LocalDateTime; -import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; @Entity @Table(name = "funders") -public class FunderDbEntry { +public class FunderDbEntry implements Serializable { + + private static final long serialVersionUID = 1290088460508203016L; @Id + @Column(name = "id") private String id; - private String name; - private String shortname; - private String jurisdiction; - private String websiteurl; - private String policy; - private Date registrationdate; - private Date lastupdatedate; - @OneToMany( - cascade = { CascadeType.PERSIST, CascadeType.MERGE }, - fetch = FetchType.LAZY ) - @JoinTable( - name = "funder_identity", - joinColumns = @JoinColumn(name = "funder"), - inverseJoinColumns = @JoinColumn(name = "pid")) - private Set pids; + @Column(name = "legalshortname") + private String legalShortName; - @OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY) - private Set fundingpaths; + @Column(name = "legalname") + private String legalName; - public FunderDbEntry() {} + @Column(name = "websiteurl") + private String websiteUrl; + + @Column(name = "logourl") + private String logoUrl; + + @Column(name = "country") + private String country; + + @Column(name = "registrationdate") + private LocalDateTime registrationDate; + + @Column(name = "registered") + private boolean registered; public String getId() { return id; @@ -42,75 +47,60 @@ public class FunderDbEntry { this.id = id; } - public String getName() { - return name; + public String getLegalShortName() { + return legalShortName; } - public void setName(final String name) { - this.name = name; + public void setLegalShortName(final String legalShortName) { + this.legalShortName = legalShortName; } - public String getShortname() { - return shortname; + public String getLegalName() { + return legalName; } - public void setShortname(final String shortname) { - this.shortname = shortname; + public void setLegalName(final String legalName) { + this.legalName = legalName; } - public String getJurisdiction() { - return jurisdiction; + public String getWebsiteUrl() { + return websiteUrl; } - public void setJurisdiction(final String jurisdiction) { - this.jurisdiction = jurisdiction; + public void setWebsiteUrl(final String websiteUrl) { + this.websiteUrl = websiteUrl; } - public String getWebsiteurl() { - return websiteurl; + public String getLogoUrl() { + return logoUrl; } - public void setWebsiteurl(final String websiteurl) { - this.websiteurl = websiteurl; + public void setLogoUrl(final String logoUrl) { + this.logoUrl = logoUrl; } - public String getPolicy() { - return policy; + public String getCountry() { + return country; } - public void setPolicy(final String policy) { - this.policy = policy; + public void setCountry(final String country) { + this.country = country; } - public Date getRegistrationdate() { - return registrationdate; + public LocalDateTime getRegistrationDate() { + return registrationDate; } - public void setRegistrationdate(final Date registrationdate) { - this.registrationdate = registrationdate; + public void setRegistrationDate(final LocalDateTime registrationDate) { + this.registrationDate = registrationDate; } - public Date getLastupdatedate() { - return lastupdatedate; + public boolean isRegistered() { + return registered; } - public void setLastupdatedate(final Date lastupdatedate) { - this.lastupdatedate = lastupdatedate; + public void setRegistered(final boolean registered) { + this.registered = registered; } - public Set getPids() { - return pids; - } - - public void setPids(final Set pids) { - this.pids = pids; - } - - public Set getFundingpaths() { - return fundingpaths; - } - - public void setFundingpaths(final Set fundingpaths) { - this.fundingpaths = fundingpaths; - } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java new file mode 100644 index 00000000..2a79306b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFunderDbEntry.java @@ -0,0 +1,118 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.sql.Date; +import java.util.Set; + +import javax.persistence.CascadeType; +import javax.persistence.FetchType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.OneToMany; + +import eu.dnetlib.openaire.dsm.domain.db.IdentityDbEntry; + +// @Entity +// @Table(name = "funders") +@Deprecated +public class OldFunderDbEntry { + + @Id + private String id; + private String name; + private String shortname; + private String jurisdiction; + private String websiteurl; + private String policy; + private Date registrationdate; + private Date lastupdatedate; + + @OneToMany(cascade = { CascadeType.PERSIST, CascadeType.MERGE }, fetch = FetchType.LAZY) + @JoinTable(name = "funder_identity", joinColumns = @JoinColumn(name = "funder"), inverseJoinColumns = @JoinColumn(name = "pid")) + private Set pids; + + @OneToMany(mappedBy = "funderid", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + private Set fundingpaths; + + public OldFunderDbEntry() {} + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getShortname() { + return shortname; + } + + public void setShortname(final String shortname) { + this.shortname = shortname; + } + + public String getJurisdiction() { + return jurisdiction; + } + + public void setJurisdiction(final String jurisdiction) { + this.jurisdiction = jurisdiction; + } + + public String getWebsiteurl() { + return websiteurl; + } + + public void setWebsiteurl(final String websiteurl) { + this.websiteurl = websiteurl; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(final String policy) { + this.policy = policy; + } + + public Date getRegistrationdate() { + return registrationdate; + } + + public void setRegistrationdate(final Date registrationdate) { + this.registrationdate = registrationdate; + } + + public Date getLastupdatedate() { + return lastupdatedate; + } + + public void setLastupdatedate(final Date lastupdatedate) { + this.lastupdatedate = lastupdatedate; + } + + public Set getPids() { + return pids; + } + + public void setPids(final Set pids) { + this.pids = pids; + } + + public Set getFundingpaths() { + return fundingpaths; + } + + public void setFundingpaths(final Set fundingpaths) { + this.fundingpaths = fundingpaths; + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java similarity index 88% rename from apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java rename to apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java index 2f7dc534..d5711b66 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FundingPathDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/OldFundingPathDbEntry.java @@ -1,12 +1,15 @@ package eu.dnetlib.openaire.funders.domain.db; -import javax.persistence.*; +import javax.persistence.Column; +import javax.persistence.Id; +import javax.persistence.Transient; import com.fasterxml.jackson.annotation.JsonIgnore; -@Entity -@Table(name = "fundingpaths") -public class FundingPathDbEntry { +// @Entity +// @Table(name = "fundingpaths") +@Deprecated +public class OldFundingPathDbEntry { @JsonIgnore @Transient @@ -28,7 +31,7 @@ public class FundingPathDbEntry { private String funderid; - public FundingPathDbEntry() {} + public OldFundingPathDbEntry() {} public String getDnetresourceidentifier() { return dnetresourceidentifier; diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql new file mode 100644 index 00000000..db0178bf --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -0,0 +1,17 @@ +ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean; + +CREATE VIEW funders_view AS SELECT + o.id AS id, + o.legalshortname AS legalshortname, + o.legalname AS legalname, + o.websiteurl AS websiteurl, + o.logourl AS logourl, + o.country AS country, + o.dateofcollection AS registrationdate, + o.registered_funder AS registered +FROM + dsm_organizations o + JOIN dsm_service_organization so ON (o.id = so.organization) + JOIN dsm_services s ON (so.service = s.id) + JOIN projects p ON (p.collectedfrom = s.id) +GROUP BY o.id; \ No newline at end of file diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java index 723feafb..51e0ecff 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/ExtendedFunderDetails.java @@ -4,6 +4,7 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonAutoDetect; +@Deprecated @JsonAutoDetect public class ExtendedFunderDetails extends FunderDetails { diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java index 863a443b..0f234854 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/funders/FundingStream.java @@ -1,5 +1,6 @@ package eu.dnetlib.openaire.exporter.model.funders; +@Deprecated public class FundingStream { private String id; From 3eed737bad35a3b514008f81c9d6837b835d2226 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 19 Feb 2024 10:57:39 +0100 Subject: [PATCH 2/7] pids --- .../funders/FundersApiController.java | 4 +- .../funders/domain/db/FunderDbEntry.java | 53 ++++++++++++++++--- .../openaire/funders/domain/db/FunderPid.java | 29 ++++++++++ .../src/main/resources/application.properties | 2 +- .../src/main/resources/sql/funders-schema.sql | 13 +++-- 5 files changed, 85 insertions(+), 16 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index e1e16177..0bc8f9b9 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -29,7 +29,7 @@ public class FundersApiController extends AbstractExporterController { @Autowired private FunderDao fDao; - @RequestMapping(value = "/funders", produces = { + @RequestMapping(value = "/funders/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, registration date") @@ -56,7 +56,7 @@ public class FundersApiController extends AbstractExporterController { return fDao.findFunder(id); } - @RequestMapping(value = "/funder/ids", produces = { + @RequestMapping(value = "/funders/{page}/{size}/ids", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "get the list of funder ids", description = "get the list of funder ids") diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index 0d66ed8f..69c207da 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -1,15 +1,30 @@ package eu.dnetlib.openaire.funders.domain.db; import java.io.Serializable; -import java.time.LocalDateTime; +import java.time.LocalDate; +import java.util.List; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; +import javax.persistence.Transient; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.vladmihalcea.hibernate.type.array.StringArrayType; +import com.vladmihalcea.hibernate.type.json.JsonBinaryType; +import com.vladmihalcea.hibernate.type.json.JsonStringType; @Entity -@Table(name = "funders") +@Table(name = "funders_view") +@TypeDefs({ + @TypeDef(name = "string-array", typeClass = StringArrayType.class), + @TypeDef(name = "json", typeClass = JsonStringType.class), + @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) +}) public class FunderDbEntry implements Serializable { private static final long serialVersionUID = 1290088460508203016L; @@ -34,10 +49,17 @@ public class FunderDbEntry implements Serializable { private String country; @Column(name = "registrationdate") - private LocalDateTime registrationDate; + private LocalDate registrationDate; @Column(name = "registered") - private boolean registered; + private Boolean registered; + + @Type(type = "json") + @Column(name = "pids", columnDefinition = "jsonb") + private List pids; + + @Transient + private List aggregationDates; public String getId() { return id; @@ -87,20 +109,35 @@ public class FunderDbEntry implements Serializable { this.country = country; } - public LocalDateTime getRegistrationDate() { + public LocalDate getRegistrationDate() { return registrationDate; } - public void setRegistrationDate(final LocalDateTime registrationDate) { + public void setRegistrationDate(final LocalDate registrationDate) { this.registrationDate = registrationDate; } - public boolean isRegistered() { + public Boolean getRegistered() { return registered; } - public void setRegistered(final boolean registered) { + public void setRegistered(final Boolean registered) { this.registered = registered; } + public List getAggregationDates() { + return aggregationDates; + } + + public void setAggregationDates(final List aggregationDates) { + this.aggregationDates = aggregationDates; + } + + public List getPids() { + return pids; + } + + public void setPids(final List pids) { + this.pids = pids; + } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java new file mode 100644 index 00000000..bf058cb4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderPid.java @@ -0,0 +1,29 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.io.Serializable; + +public class FunderPid implements Serializable { + + private static final long serialVersionUID = 2145493560459874509L; + + private String type; + + private String value; + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(final String value) { + this.value = value; + } + +} diff --git a/apps/dnet-exporter-api/src/main/resources/application.properties b/apps/dnet-exporter-api/src/main/resources/application.properties index 3aeb5b58..f5ec907b 100644 --- a/apps/dnet-exporter-api/src/main/resources/application.properties +++ b/apps/dnet-exporter-api/src/main/resources/application.properties @@ -27,7 +27,7 @@ openaire.exporter.enable.dsm = true openaire.exporter.enable.community = true openaire.exporter.enable.community.import = false openaire.exporter.enable.context = true -openaire.exporter.enable.funders = false +openaire.exporter.enable.funders = true openaire.exporter.enable.project = true openaire.exporter.enable.info = true diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index db0178bf..36fe9bd6 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -8,10 +8,13 @@ CREATE VIEW funders_view AS SELECT o.logourl AS logourl, o.country AS country, o.dateofcollection AS registrationdate, - o.registered_funder AS registered + o.registered_funder AS registered, + CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids FROM dsm_organizations o - JOIN dsm_service_organization so ON (o.id = so.organization) - JOIN dsm_services s ON (so.service = s.id) - JOIN projects p ON (p.collectedfrom = s.id) -GROUP BY o.id; \ No newline at end of file + JOIN dsm_service_organization so ON (o.id = so.organization) + JOIN dsm_services s ON (so.service = s.id) + JOIN projects p ON (p.collectedfrom = s.id) + LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization) + LEFT OUTER JOIN dsm_identities pids ON (opids.pid = pids.pid) +GROUP BY o.id; From fc750633b920aad381590c95661eef640fd8d47b Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 19 Feb 2024 12:37:50 +0100 Subject: [PATCH 3/7] aggregation dates --- .../dnetlib/openaire/funders/FunderDao.java | 55 ++++++++++++++++++- .../funders/FundersApiController.java | 14 ++++- .../funders/domain/db/FunderDatasource.java | 39 +++++++++++++ .../funders/domain/db/FunderDbEntry.java | 12 ++++ .../src/main/resources/sql/funders-schema.sql | 1 + 5 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java index 5cde4c94..d410091d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java @@ -1,14 +1,25 @@ package eu.dnetlib.openaire.funders; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; +import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; +import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; +import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; +import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; +import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; @Component @@ -18,14 +29,24 @@ public class FunderDao { @Autowired private FunderRepository funderRepository; + @Autowired + private MongoLoggerClient mongoLoggerClient; + + private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + private static final Log log = LogFactory.getLog(FunderDao.class); + public FunderDbEntry findFunder(final String funderId) throws FundersApiException { - return funderRepository.findById(funderId).orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId)); + return funderRepository.findById(funderId) + .map(this::addAggregationHistory) + .orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId)); } public List listFunders(final int page, final int size) throws FundersApiException { return funderRepository.findAll(PageRequest.of(page, size)) .getContent() .stream() + .map(this::addAggregationHistory) .collect(Collectors.toList()); } @@ -37,4 +58,36 @@ public class FunderDao { .collect(Collectors.toList()); } + public long count() { + return funderRepository.count(); + } + + private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) { + + final List dates = funder.getDatasources() + .stream() + .map(FunderDatasource::getId) + .map(id -> { + try { + return mongoLoggerClient.getAggregationHistoryV2(id); + } catch (final DsmApiException e) { + log.error("Error retrieving the aggregation history"); + // throw new RuntimeException(e); + return new ArrayList(); + } + }) + .flatMap(List::stream) + .filter(AggregationInfo::isCompletedSuccessfully) + .filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) + .map(AggregationInfo::getDate) + .distinct() + .map(s -> LocalDate.parse(s, DATEFORMATTER)) + .sorted(Comparator.reverseOrder()) + .toList(); + + funder.setAggregationDates(dates); + + return funder; + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index 0bc8f9b9..703e3b1b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -32,7 +32,7 @@ public class FundersApiController extends AbstractExporterController { @RequestMapping(value = "/funders/{page}/{size}", produces = { "application/json" }, method = RequestMethod.GET) - @Operation(summary = "get basic information about funders", description = "basic information about funders: id, name, shortname, registration date") + @Operation(summary = "get a page of funders", description = "get a page of funders") @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") @@ -71,4 +71,16 @@ public class FundersApiController extends AbstractExporterController { return fDao.listFunderIds(page, size); } + @RequestMapping(value = "/funders/count", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "count the funders", description = "count the funders") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public long countFunders() throws FundersApiException { + return fDao.count(); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java new file mode 100644 index 00000000..96556de0 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDatasource.java @@ -0,0 +1,39 @@ +package eu.dnetlib.openaire.funders.domain.db; + +import java.io.Serializable; + +public class FunderDatasource implements Serializable { + + private static final long serialVersionUID = 2145493560459874509L; + + private String id; + + private String name; + + private String type; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index 69c207da..4a49713d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -58,6 +58,10 @@ public class FunderDbEntry implements Serializable { @Column(name = "pids", columnDefinition = "jsonb") private List pids; + @Type(type = "json") + @Column(name = "datasources", columnDefinition = "jsonb") + private List datasources; + @Transient private List aggregationDates; @@ -140,4 +144,12 @@ public class FunderDbEntry implements Serializable { public void setPids(final List pids) { this.pids = pids; } + + public List getDatasources() { + return datasources; + } + + public void setDatasources(final List datasources) { + this.datasources = datasources; + } } diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 36fe9bd6..1e9ee820 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -9,6 +9,7 @@ CREATE VIEW funders_view AS SELECT o.country AS country, o.dateofcollection AS registrationdate, o.registered_funder AS registered, + CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources, CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids FROM dsm_organizations o From e5a9ee0b217f6469db6ff7ad5fb3d0fb7935b0b4 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 19 Feb 2024 16:06:34 +0100 Subject: [PATCH 4/7] refactoring using temp file --- apps/dnet-exporter-api/pom.xml | 4 + .../dnetlib/openaire/funders/FunderDao.java | 93 ------------ .../openaire/funders/FunderService.java | 139 ++++++++++++++++++ .../funders/FundersApiController.java | 78 +++++----- .../src/main/resources/application.properties | 1 + .../src/main/resources/sql/funders-schema.sql | 4 + .../funders/FunderContextClientTest.java | 4 +- 7 files changed, 183 insertions(+), 140 deletions(-) delete mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index 04ac9d4b..8f7200cb 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -113,6 +113,10 @@ com.fasterxml.jackson.datatype jackson-datatype-joda + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + joda-time joda-time diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java deleted file mode 100644 index d410091d..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderDao.java +++ /dev/null @@ -1,93 +0,0 @@ -package eu.dnetlib.openaire.funders; - -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.data.domain.PageRequest; -import org.springframework.stereotype.Component; - -import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; -import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; -import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; -import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; -import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; -import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; - -@Component -@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") -public class FunderDao { - - @Autowired - private FunderRepository funderRepository; - - @Autowired - private MongoLoggerClient mongoLoggerClient; - - private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - private static final Log log = LogFactory.getLog(FunderDao.class); - - public FunderDbEntry findFunder(final String funderId) throws FundersApiException { - return funderRepository.findById(funderId) - .map(this::addAggregationHistory) - .orElseThrow(() -> new FundersApiException("Funder not found. ID: " + funderId)); - } - - public List listFunders(final int page, final int size) throws FundersApiException { - return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(this::addAggregationHistory) - .collect(Collectors.toList()); - } - - public List listFunderIds(final int page, final int size) throws FundersApiException { - return funderRepository.findAll(PageRequest.of(page, size)) - .getContent() - .stream() - .map(FunderDbEntry::getId) - .collect(Collectors.toList()); - } - - public long count() { - return funderRepository.count(); - } - - private FunderDbEntry addAggregationHistory(final FunderDbEntry funder) { - - final List dates = funder.getDatasources() - .stream() - .map(FunderDatasource::getId) - .map(id -> { - try { - return mongoLoggerClient.getAggregationHistoryV2(id); - } catch (final DsmApiException e) { - log.error("Error retrieving the aggregation history"); - // throw new RuntimeException(e); - return new ArrayList(); - } - }) - .flatMap(List::stream) - .filter(AggregationInfo::isCompletedSuccessfully) - .filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) - .map(AggregationInfo::getDate) - .distinct() - .map(s -> LocalDate.parse(s, DATEFORMATTER)) - .sorted(Comparator.reverseOrder()) - .toList(); - - funder.setAggregationDates(dates); - - return funder; - } - -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java new file mode 100644 index 00000000..bdf2ff48 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -0,0 +1,139 @@ +package eu.dnetlib.openaire.funders; + +import java.io.File; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient; +import eu.dnetlib.openaire.exporter.exceptions.DsmApiException; +import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; +import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; +import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; +import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; + +@Component +@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") +public class FunderService { + + private static final String TEMP_FILE_SUFFIX = ".funds.tmp"; + + @Autowired + private FunderRepository funderRepository; + + @Autowired + private MongoLoggerClient mongoLoggerClient; + + private File tempFile; + + private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + private static final Log log = LogFactory.getLog(FunderService.class); + + @PostConstruct + public void init() { + for (final File f : new File("/tmp").listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) { + deleteFile(f); + } + + new Thread(this::updateFunders).start(); + } + + private void deleteFile(final File f) { + if (f != null && f.exists()) { + log.info("Deleting file: " + f.getAbsolutePath()); + f.delete(); + } + } + + @Scheduled(cron = "${openaire.exporter.funders.cron}") + public void updateFunders() { + try { + final ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + + final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, new File("/tmp")); + + log.info("Generating funders file: " + tmp.getAbsolutePath()); + + try (final FileWriter writer = new FileWriter(tmp)) { + writer.write("["); + boolean first = true; + for (final FunderDbEntry funder : funderRepository.findAll()) { + log.info(" - adding: " + funder.getId()); + // addAggregationHistory(funder); + + if (first) { + first = false; + } else { + writer.write(","); + } + writer.write(mapper.writeValueAsString(funder)); + } + writer.write("]"); + log.info("Publish funders file: " + tmp.getAbsolutePath()); + + synchronized (this) { + deleteFile(tempFile); + setTempFile(tmp); + } + } + } catch (final IOException e) { + log.error("Error generating funders file", e); + throw new RuntimeException("Error generating funders file", e); + } + } + + private void addAggregationHistory(final FunderDbEntry funder) { + + final List dates = funder.getDatasources() + .stream() + .map(FunderDatasource::getId) + .map(id -> { + try { + return mongoLoggerClient.getAggregationHistoryV2(id); + } catch (final DsmApiException e) { + log.error("Error retrieving the aggregation history", e); + throw new RuntimeException("Error retrieving the aggregation history", e); + } + }) + .flatMap(List::stream) + .filter(AggregationInfo::isCompletedSuccessfully) + .filter(info -> info.getAggregationStage() == AggregationStage.TRANSFORM) + .map(AggregationInfo::getDate) + .distinct() + .map(s -> LocalDate.parse(s, DATEFORMATTER)) + .sorted(Comparator.reverseOrder()) + .collect(Collectors.toList()); + + funder.setAggregationDates(dates); + } + + public synchronized File getTempFile() { + return tempFile; + } + + public void setTempFile(final File tempFile) { + this.tempFile = tempFile; + } + +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java index 703e3b1b..f4b0bf8d 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FundersApiController.java @@ -1,18 +1,25 @@ package eu.dnetlib.openaire.funders; -import java.util.List; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.exporter.exceptions.FundersApiException; -import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; @@ -27,60 +34,41 @@ import io.swagger.v3.oas.annotations.tags.Tag; public class FundersApiController extends AbstractExporterController { @Autowired - private FunderDao fDao; + private FunderService service; - @RequestMapping(value = "/funders/{page}/{size}", produces = { + private static final Log log = LogFactory.getLog(FundersApiController.class); + + @RequestMapping(value = "/funders", produces = { "application/json" }, method = RequestMethod.GET) - @Operation(summary = "get a page of funders", description = "get a page of funders") + @Operation(summary = "get all funders", description = "get all funders") @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public List getFunders( - @PathVariable final int page, - @PathVariable final int size) throws FundersApiException { + public void getFunders(final HttpServletResponse res) throws FundersApiException { - return fDao.listFunders(page, size); - } + res.setContentType(MediaType.APPLICATION_JSON_VALUE); - @RequestMapping(value = "/funder/{id}", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "get the funder details", description = "complete funder information") - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public FunderDbEntry getFunderDetails(@PathVariable final String id) throws FundersApiException { - return fDao.findFunder(id); - } + final File file = service.getTempFile(); - @RequestMapping(value = "/funders/{page}/{size}/ids", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "get the list of funder ids", description = "get the list of funder ids") - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public List listFunderIds( - @PathVariable final int page, - @PathVariable final int size) throws FundersApiException { + if (file == null) { + log.error("Missing temp file (NULL)"); + throw new FundersApiException("Missing temp file (NULL)"); + } - return fDao.listFunderIds(page, size); - } + if (!file.exists()) { + log.error("Missing temp file " + service.getTempFile()); + throw new FundersApiException("Missing temp file " + service.getTempFile()); + } - @RequestMapping(value = "/funders/count", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "count the funders", description = "count the funders") - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public long countFunders() throws FundersApiException { - return fDao.count(); + try (final InputStream in = new FileInputStream(file); OutputStream out = res.getOutputStream()) { + IOUtils.copy(in, out); + return; + } catch (final Exception e) { + log.error("Error reading file " + service.getTempFile(), e); + throw new FundersApiException("Error reading file " + service.getTempFile(), e); + } } } diff --git a/apps/dnet-exporter-api/src/main/resources/application.properties b/apps/dnet-exporter-api/src/main/resources/application.properties index f5ec907b..19150128 100644 --- a/apps/dnet-exporter-api/src/main/resources/application.properties +++ b/apps/dnet-exporter-api/src/main/resources/application.properties @@ -36,3 +36,4 @@ openaire.exporter.cache.ttl = 43200000 maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-exporter-api/effective-pom.xml +openaire.exporter.funders.cron = 0 30 2 * * * diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 1e9ee820..a346911c 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -1,5 +1,9 @@ ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean; + + +-- TODO OCCORRE METTERE DELLE DISTINCT + CREATE VIEW funders_view AS SELECT o.id AS id, o.legalshortname AS legalshortname, diff --git a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java index 8869dba4..b2fef0e1 100644 --- a/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java +++ b/apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/funders/FunderContextClientTest.java @@ -20,11 +20,11 @@ import eu.dnetlib.openaire.exporter.model.funders.FunderDetails; public class FunderContextClientTest { - private FunderDao fDao; + private FunderService fDao; @BeforeEach public void setUp() { - fDao = new FunderDao(); + fDao = new FunderService(); } @Test From 383fc49908ec2b79713ce539316aff2bf4ae4ab2 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 20 Feb 2024 08:32:11 +0100 Subject: [PATCH 5/7] SQLDISTINCT --- .../eu/dnetlib/openaire/funders/FunderService.java | 12 +++++++++--- .../src/main/resources/sql/funders-schema.sql | 8 ++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index bdf2ff48..4a7914f6 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -42,6 +42,8 @@ public class FunderService { @Autowired private MongoLoggerClient mongoLoggerClient; + private File tempDir; + private File tempFile; private final DateTimeFormatter DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); @@ -50,7 +52,10 @@ public class FunderService { @PostConstruct public void init() { - for (final File f : new File("/tmp").listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) { + + tempDir = new File(System.getProperty("java.io.tmpdir", "/tmp")); + + for (final File f : tempDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(TEMP_FILE_SUFFIX))) { deleteFile(f); } @@ -71,7 +76,7 @@ public class FunderService { mapper.registerModule(new JavaTimeModule()); mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, new File("/tmp")); + final File tmp = File.createTempFile("funders-api-", TEMP_FILE_SUFFIX, tempDir); log.info("Generating funders file: " + tmp.getAbsolutePath()); @@ -80,7 +85,7 @@ public class FunderService { boolean first = true; for (final FunderDbEntry funder : funderRepository.findAll()) { log.info(" - adding: " + funder.getId()); - // addAggregationHistory(funder); + addAggregationHistory(funder); if (first) { first = false; @@ -123,6 +128,7 @@ public class FunderService { .distinct() .map(s -> LocalDate.parse(s, DATEFORMATTER)) .sorted(Comparator.reverseOrder()) + .limit(10) .collect(Collectors.toList()); funder.setAggregationDates(dates); diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index a346911c..a2cf0dd3 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -1,9 +1,5 @@ ALTER TABLE dsm_organizations ADD COLUMN registered_funder boolean; - - --- TODO OCCORRE METTERE DELLE DISTINCT - CREATE VIEW funders_view AS SELECT o.id AS id, o.legalshortname AS legalshortname, @@ -13,8 +9,8 @@ CREATE VIEW funders_view AS SELECT o.country AS country, o.dateofcollection AS registrationdate, o.registered_funder AS registered, - CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources, - CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids + CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources, + CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids FROM dsm_organizations o JOIN dsm_service_organization so ON (o.id = so.organization) From 817864942d536e7fcbfc9338cb13269a9e9595c9 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 20 Feb 2024 09:37:20 +0100 Subject: [PATCH 6/7] add a SQL GRANT --- .../java/eu/dnetlib/openaire/funders/FunderService.java | 8 +++----- .../src/main/resources/sql/funders-schema.sql | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index 4a7914f6..73eaba66 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -97,10 +97,8 @@ public class FunderService { writer.write("]"); log.info("Publish funders file: " + tmp.getAbsolutePath()); - synchronized (this) { - deleteFile(tempFile); - setTempFile(tmp); - } + deleteFile(tempFile); + setTempFile(tmp); } } catch (final IOException e) { log.error("Error generating funders file", e); @@ -134,7 +132,7 @@ public class FunderService { funder.setAggregationDates(dates); } - public synchronized File getTempFile() { + public File getTempFile() { return tempFile; } diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index a2cf0dd3..9b857057 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -19,3 +19,5 @@ FROM LEFT OUTER JOIN dsm_organizationpids opids ON (o.id = opids.organization) LEFT OUTER JOIN dsm_identities pids ON (opids.pid = pids.pid) GROUP BY o.id; + +GRANT ALL ON funders_view TO dnetapi; From 55ca95a109d418f34ea6baf0612d6352f1e25c7a Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 22 Feb 2024 10:05:45 +0100 Subject: [PATCH 7/7] fixed a compatibility problem with postgres 9.3 --- .../openaire/funders/FunderService.java | 44 ++++++++++++++- .../funders/domain/db/FunderDbEntry.java | 53 +++++++++++++------ .../src/main/resources/sql/funders-schema.sql | 6 ++- 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index 73eaba66..756259c9 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -3,15 +3,17 @@ package eu.dnetlib.openaire.funders; import java.io.File; import java.io.FileWriter; import java.io.FilenameFilter; -import java.io.IOException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.PostConstruct; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,12 +31,14 @@ import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FunderPid; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") public class FunderService { private static final String TEMP_FILE_SUFFIX = ".funds.tmp"; + private static final String SEPARATOR = "@=@"; @Autowired private FunderRepository funderRepository; @@ -85,6 +89,42 @@ public class FunderService { boolean first = true; for (final FunderDbEntry funder : funderRepository.findAll()) { log.info(" - adding: " + funder.getId()); + + // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) + + final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 3) + .map(arr -> { + final FunderDatasource ds = new FunderDatasource(); + ds.setId(arr[0].trim()); + ds.setName(arr[1].trim()); + ds.setType(arr[2].trim()); + return ds; + }) + .filter(ds -> StringUtils.isNotBlank(ds.getId())) + .collect(Collectors.toList()); + + funder.setDatasources(datasources); + + final List pids = Arrays.stream(funder.getPidsPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 2) + .map(arr -> { + final FunderPid pid = new FunderPid(); + pid.setType(arr[0].trim()); + pid.setValue(arr[1].trim()); + return pid; + }) + .filter(pid -> StringUtils.isNotBlank(pid.getValue())) + .collect(Collectors.toList()); + + funder.setPids(pids); + + // END PATCH + addAggregationHistory(funder); if (first) { @@ -100,7 +140,7 @@ public class FunderService { deleteFile(tempFile); setTempFile(tmp); } - } catch (final IOException e) { + } catch (final Throwable e) { log.error("Error generating funders file", e); throw new RuntimeException("Error generating funders file", e); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index 4a49713d..1a3219c8 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -2,6 +2,7 @@ package eu.dnetlib.openaire.funders.domain.db; import java.io.Serializable; import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import javax.persistence.Column; @@ -14,16 +15,13 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.hibernate.annotations.TypeDefs; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.vladmihalcea.hibernate.type.array.StringArrayType; -import com.vladmihalcea.hibernate.type.json.JsonBinaryType; -import com.vladmihalcea.hibernate.type.json.JsonStringType; @Entity @Table(name = "funders_view") @TypeDefs({ - @TypeDef(name = "string-array", typeClass = StringArrayType.class), - @TypeDef(name = "json", typeClass = JsonStringType.class), - @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) + @TypeDef(name = "string-array", typeClass = StringArrayType.class) }) public class FunderDbEntry implements Serializable { @@ -54,13 +52,21 @@ public class FunderDbEntry implements Serializable { @Column(name = "registered") private Boolean registered; - @Type(type = "json") - @Column(name = "pids", columnDefinition = "jsonb") - private List pids; + @JsonIgnore + @Type(type = "string-array") + @Column(name = "pids", columnDefinition = "text[]") + private String[] pidsPostgres; - @Type(type = "json") - @Column(name = "datasources", columnDefinition = "jsonb") - private List datasources; + @Transient + private List pids = new ArrayList(); + + @JsonIgnore + @Type(type = "string-array") + @Column(name = "datasources", columnDefinition = "text[]") + private String[] datasourcesPostgres; + + @Transient + private List datasources = new ArrayList(); @Transient private List aggregationDates; @@ -129,12 +135,12 @@ public class FunderDbEntry implements Serializable { this.registered = registered; } - public List getAggregationDates() { - return aggregationDates; + public String[] getPidsPostgres() { + return pidsPostgres; } - public void setAggregationDates(final List aggregationDates) { - this.aggregationDates = aggregationDates; + public void setPidsPostgres(final String[] pidsPostgres) { + this.pidsPostgres = pidsPostgres; } public List getPids() { @@ -145,6 +151,14 @@ public class FunderDbEntry implements Serializable { this.pids = pids; } + public String[] getDatasourcesPostgres() { + return datasourcesPostgres; + } + + public void setDatasourcesPostgres(final String[] datasourcesPostgres) { + this.datasourcesPostgres = datasourcesPostgres; + } + public List getDatasources() { return datasources; } @@ -152,4 +166,13 @@ public class FunderDbEntry implements Serializable { public void setDatasources(final List datasources) { this.datasources = datasources; } + + public List getAggregationDates() { + return aggregationDates; + } + + public void setAggregationDates(final List aggregationDates) { + this.aggregationDates = aggregationDates; + } + } diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 9b857057..9e733f32 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -9,8 +9,8 @@ CREATE VIEW funders_view AS SELECT o.country AS country, o.dateofcollection AS registrationdate, o.registered_funder AS registered, - CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources, - CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids + array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources, + array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids FROM dsm_organizations o JOIN dsm_service_organization so ON (o.id = so.organization) @@ -21,3 +21,5 @@ FROM GROUP BY o.id; GRANT ALL ON funders_view TO dnetapi; + +