D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 10 Projects 2 Releases 2 Wiki Activity

Merge pull request 'organizations pids' (#53) from organization_pids into master 

LGTM
This commit is contained in:
Claudio Atzori 2020-11-11 14:08:25 +01:00
parent 18d9aad70c 40160d171f
commit e3d3481fb9
2 changed files with 36 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java
View File

@ -38,7 +38,11 @@ import java.io.IOException;
import java.sql.Array;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
@ -174,7 +178,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
execute(sqlFile, producer, oaf -> true);
}
public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer,
public void execute(final String sqlFile,
final Function<ResultSet, List<Oaf>> producer,
final Predicate<Oaf> predicate)
throws Exception {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
@ -198,8 +203,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
ds
.setOriginalId(
Arrays
.asList(
(String[]) rs.getArray("identities").getArray())
.asList((String[]) rs.getArray("identities").getArray())
.stream()
.filter(StringUtils::isNotBlank)
.collect(Collectors.toList()));
@ -250,11 +254,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
ds
.setJournal(
journal(
rs.getString("officialname"),
rs.getString("issnPrinted"),
rs.getString("issnOnline"),
rs.getString("issnLinking"),
info)); // Journal
rs.getString("officialname"), rs.getString("issnPrinted"), rs.getString("issnOnline"),
rs.getString("issnLinking"), info)); // Journal
ds.setDataInfo(info);
ds.setLastupdatetimestamp(lastUpdateTimestamp);
@ -332,7 +333,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname")));
o.setPid(new ArrayList<>());
o.setPid(prepareListOfStructProps(rs.getArray("pid"), info));
o.setDateofcollection(asString(rs.getDate("dateofcollection")));
o.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
o.setExtraInfo(new ArrayList<>()); // Values not present in the DB

32
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql
View File

@ -24,12 +24,30 @@ SELECT
d.officialname AS collectedfromname,
o.country || '@@@dnet:countries' AS country,
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
ARRAY[]::text[] AS pid
array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype), NULL) AS pid
FROM dsm_organizations o
LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom)
LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id)
LEFT OUTER JOIN dsm_identities i ON (i.pid = p.pid)
GROUP BY
o.id,
o.legalshortname,
o.legalname,
o.websiteurl,
o.logourl,
o.ec_legalbody,
o.ec_legalperson,
o.ec_nonprofit,
o.ec_researchorganization,
o.ec_highereducation,
o.ec_internationalorganizationeurinterests,
o.ec_internationalorganization,
o.ec_enterprise,
o.ec_smevalidated,
o.ec_nutscode,
o.dateofcollection,
o.lastupdate,
o.trust,
d.id,
d.officialname,
o.country