From 55ca95a109d418f34ea6baf0612d6352f1e25c7a Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 22 Feb 2024 10:05:45 +0100 Subject: [PATCH] fixed a compatibility problem with postgres 9.3 --- .../openaire/funders/FunderService.java | 44 ++++++++++++++- .../funders/domain/db/FunderDbEntry.java | 53 +++++++++++++------ .../src/main/resources/sql/funders-schema.sql | 6 ++- 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java index 73eaba66..756259c9 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java @@ -3,15 +3,17 @@ package eu.dnetlib.openaire.funders; import java.io.File; import java.io.FileWriter; import java.io.FilenameFilter; -import java.io.IOException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.PostConstruct; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,12 +31,14 @@ import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo; import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage; import eu.dnetlib.openaire.funders.domain.db.FunderDatasource; import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry; +import eu.dnetlib.openaire.funders.domain.db.FunderPid; @Component @ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true") public class FunderService { private static final String TEMP_FILE_SUFFIX = ".funds.tmp"; + private static final String SEPARATOR = "@=@"; @Autowired private FunderRepository funderRepository; @@ -85,6 +89,42 @@ public class FunderService { boolean first = true; for (final FunderDbEntry funder : funderRepository.findAll()) { log.info(" - adding: " + funder.getId()); + + // THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY) + + final List datasources = Arrays.stream(funder.getDatasourcesPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 3) + .map(arr -> { + final FunderDatasource ds = new FunderDatasource(); + ds.setId(arr[0].trim()); + ds.setName(arr[1].trim()); + ds.setType(arr[2].trim()); + return ds; + }) + .filter(ds -> StringUtils.isNotBlank(ds.getId())) + .collect(Collectors.toList()); + + funder.setDatasources(datasources); + + final List pids = Arrays.stream(funder.getPidsPostgres()) + .filter(Objects::nonNull) + .map(s -> s.split(SEPARATOR)) + .filter(arr -> arr.length == 2) + .map(arr -> { + final FunderPid pid = new FunderPid(); + pid.setType(arr[0].trim()); + pid.setValue(arr[1].trim()); + return pid; + }) + .filter(pid -> StringUtils.isNotBlank(pid.getValue())) + .collect(Collectors.toList()); + + funder.setPids(pids); + + // END PATCH + addAggregationHistory(funder); if (first) { @@ -100,7 +140,7 @@ public class FunderService { deleteFile(tempFile); setTempFile(tmp); } - } catch (final IOException e) { + } catch (final Throwable e) { log.error("Error generating funders file", e); throw new RuntimeException("Error generating funders file", e); } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java index 4a49713d..1a3219c8 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java @@ -2,6 +2,7 @@ package eu.dnetlib.openaire.funders.domain.db; import java.io.Serializable; import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; import javax.persistence.Column; @@ -14,16 +15,13 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.hibernate.annotations.TypeDefs; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.vladmihalcea.hibernate.type.array.StringArrayType; -import com.vladmihalcea.hibernate.type.json.JsonBinaryType; -import com.vladmihalcea.hibernate.type.json.JsonStringType; @Entity @Table(name = "funders_view") @TypeDefs({ - @TypeDef(name = "string-array", typeClass = StringArrayType.class), - @TypeDef(name = "json", typeClass = JsonStringType.class), - @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) + @TypeDef(name = "string-array", typeClass = StringArrayType.class) }) public class FunderDbEntry implements Serializable { @@ -54,13 +52,21 @@ public class FunderDbEntry implements Serializable { @Column(name = "registered") private Boolean registered; - @Type(type = "json") - @Column(name = "pids", columnDefinition = "jsonb") - private List pids; + @JsonIgnore + @Type(type = "string-array") + @Column(name = "pids", columnDefinition = "text[]") + private String[] pidsPostgres; - @Type(type = "json") - @Column(name = "datasources", columnDefinition = "jsonb") - private List datasources; + @Transient + private List pids = new ArrayList(); + + @JsonIgnore + @Type(type = "string-array") + @Column(name = "datasources", columnDefinition = "text[]") + private String[] datasourcesPostgres; + + @Transient + private List datasources = new ArrayList(); @Transient private List aggregationDates; @@ -129,12 +135,12 @@ public class FunderDbEntry implements Serializable { this.registered = registered; } - public List getAggregationDates() { - return aggregationDates; + public String[] getPidsPostgres() { + return pidsPostgres; } - public void setAggregationDates(final List aggregationDates) { - this.aggregationDates = aggregationDates; + public void setPidsPostgres(final String[] pidsPostgres) { + this.pidsPostgres = pidsPostgres; } public List getPids() { @@ -145,6 +151,14 @@ public class FunderDbEntry implements Serializable { this.pids = pids; } + public String[] getDatasourcesPostgres() { + return datasourcesPostgres; + } + + public void setDatasourcesPostgres(final String[] datasourcesPostgres) { + this.datasourcesPostgres = datasourcesPostgres; + } + public List getDatasources() { return datasources; } @@ -152,4 +166,13 @@ public class FunderDbEntry implements Serializable { public void setDatasources(final List datasources) { this.datasources = datasources; } + + public List getAggregationDates() { + return aggregationDates; + } + + public void setAggregationDates(final List aggregationDates) { + this.aggregationDates = aggregationDates; + } + } diff --git a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql index 9b857057..9e733f32 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql @@ -9,8 +9,8 @@ CREATE VIEW funders_view AS SELECT o.country AS country, o.dateofcollection AS registrationdate, o.registered_funder AS registered, - CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources, - CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids + array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources, + array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids FROM dsm_organizations o JOIN dsm_service_organization so ON (o.id = so.organization) @@ -21,3 +21,5 @@ FROM GROUP BY o.id; GRANT ALL ON funders_view TO dnetapi; + +