D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new_funders_api #17

Merged
michele.artini merged 7 commits from new_funders_api into master 2024-02-22 11:23:32 +01:00
Conversation 0 Commits 7 Files Changed 17 +84 -19
3 changed files with 84 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 55ca95a109 - Show all commits

44
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/FunderService.java
View File

@ -3,15 +3,17 @@ package eu.dnetlib.openaire.funders;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -29,12 +31,14 @@ import eu.dnetlib.openaire.exporter.model.dsm.AggregationInfo;
import eu.dnetlib.openaire.exporter.model.dsm.AggregationStage;
import eu.dnetlib.openaire.funders.domain.db.FunderDatasource;
import eu.dnetlib.openaire.funders.domain.db.FunderDbEntry;
import eu.dnetlib.openaire.funders.domain.db.FunderPid;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.funders", havingValue = "true")
public class FunderService {
private static final String TEMP_FILE_SUFFIX = ".funds.tmp";
private static final String SEPARATOR = "@=@";
@Autowired
private FunderRepository funderRepository;
@ -85,6 +89,42 @@ public class FunderService {
boolean first = true;
for (final FunderDbEntry funder : funderRepository.findAll()) {
log.info(" - adding: " + funder.getId());
// THIS PATCH IS NECESSARY FOR COMPATIBILITY WITH POSTGRES 9.3 (PARTIAL SUPPORT OF THE JSON LIBRARY)
final List<FunderDatasource> datasources = Arrays.stream(funder.getDatasourcesPostgres())
.filter(Objects::nonNull)
.map(s -> s.split(SEPARATOR))
.filter(arr -> arr.length == 3)
.map(arr -> {
final FunderDatasource ds = new FunderDatasource();
ds.setId(arr[0].trim());
ds.setName(arr[1].trim());
ds.setType(arr[2].trim());
return ds;
})
.filter(ds -> StringUtils.isNotBlank(ds.getId()))
.collect(Collectors.toList());
funder.setDatasources(datasources);
final List<FunderPid> pids = Arrays.stream(funder.getPidsPostgres())
.filter(Objects::nonNull)
.map(s -> s.split(SEPARATOR))
.filter(arr -> arr.length == 2)
.map(arr -> {
final FunderPid pid = new FunderPid();
pid.setType(arr[0].trim());
pid.setValue(arr[1].trim());
return pid;
})
.filter(pid -> StringUtils.isNotBlank(pid.getValue()))
.collect(Collectors.toList());
funder.setPids(pids);
// END PATCH
addAggregationHistory(funder);
if (first) {
@ -100,7 +140,7 @@ public class FunderService {
deleteFile(tempFile);
setTempFile(tmp);
}
} catch (final IOException e) {
} catch (final Throwable e) {
log.error("Error generating funders file", e);
throw new RuntimeException("Error generating funders file", e);
}

53
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/funders/domain/db/FunderDbEntry.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.openaire.funders.domain.db;
import java.io.Serializable;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import javax.persistence.Column;
@ -14,16 +15,13 @@ import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.hibernate.annotations.TypeDefs;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.vladmihalcea.hibernate.type.array.StringArrayType;
import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
import com.vladmihalcea.hibernate.type.json.JsonStringType;
@Entity
@Table(name = "funders_view")
@TypeDefs({
@TypeDef(name = "string-array", typeClass = StringArrayType.class),
@TypeDef(name = "json", typeClass = JsonStringType.class),
@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
@TypeDef(name = "string-array", typeClass = StringArrayType.class)
})
public class FunderDbEntry implements Serializable {
@ -54,13 +52,21 @@ public class FunderDbEntry implements Serializable {
@Column(name = "registered")
private Boolean registered;
@Type(type = "json")
@Column(name = "pids", columnDefinition = "jsonb")
private List<FunderPid> pids;
@JsonIgnore
@Type(type = "string-array")
@Column(name = "pids", columnDefinition = "text[]")
private String[] pidsPostgres;
@Type(type = "json")
@Column(name = "datasources", columnDefinition = "jsonb")
private List<FunderDatasource> datasources;
@Transient
private List<FunderPid> pids = new ArrayList<FunderPid>();
@JsonIgnore
@Type(type = "string-array")
@Column(name = "datasources", columnDefinition = "text[]")
private String[] datasourcesPostgres;
@Transient
private List<FunderDatasource> datasources = new ArrayList<FunderDatasource>();
@Transient
private List<LocalDate> aggregationDates;
@ -129,12 +135,12 @@ public class FunderDbEntry implements Serializable {
this.registered = registered;
}
public List<LocalDate> getAggregationDates() {
return aggregationDates;
public String[] getPidsPostgres() {
return pidsPostgres;
}
public void setAggregationDates(final List<LocalDate> aggregationDates) {
this.aggregationDates = aggregationDates;
public void setPidsPostgres(final String[] pidsPostgres) {
this.pidsPostgres = pidsPostgres;
}
public List<FunderPid> getPids() {
@ -145,6 +151,14 @@ public class FunderDbEntry implements Serializable {
this.pids = pids;
}
public String[] getDatasourcesPostgres() {
return datasourcesPostgres;
}
public void setDatasourcesPostgres(final String[] datasourcesPostgres) {
this.datasourcesPostgres = datasourcesPostgres;
}
public List<FunderDatasource> getDatasources() {
return datasources;
}
@ -152,4 +166,13 @@ public class FunderDbEntry implements Serializable {
public void setDatasources(final List<FunderDatasource> datasources) {
this.datasources = datasources;
}
public List<LocalDate> getAggregationDates() {
return aggregationDates;
}
public void setAggregationDates(final List<LocalDate> aggregationDates) {
this.aggregationDates = aggregationDates;
}
}

6
apps/dnet-exporter-api/src/main/resources/sql/funders-schema.sql
View File

@ -9,8 +9,8 @@ CREATE VIEW funders_view AS SELECT
o.country AS country,
o.dateofcollection AS registrationdate,
o.registered_funder AS registered,
CASE WHEN count(s.id) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('id', s.id,'name', s.officialname,'type',s.eosc_datasource_type)) END AS datasources,
CASE WHEN count(pids.pid) = 0 THEN '[]'::jsonb ELSE jsonb_agg(DISTINCT jsonb_build_object('type', pids.issuertype,'value', pids.pid)) END AS pids
array_agg(DISTINCT s.id||' @=@ '||s.officialname||' @=@ '||s.eosc_datasource_type) AS datasources,
array_agg(DISTINCT pids.issuertype||' @=@ '||pids.pid) AS pids
FROM
dsm_organizations o
JOIN dsm_service_organization so ON (o.id = so.organization)
@ -21,3 +21,5 @@ FROM
GROUP BY o.id;
GRANT ALL ON funders_view TO dnetapi;