sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Merge pull request 'organizations pids' (#53) from organization_pids into master 

LGTM
This commit is contained in:
Claudio Atzori 2020-11-11 14:08:25 +01:00
parent 18d9aad70c 40160d171f
commit e3d3481fb9
2 changed files with 36 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java
View File

@ -38,7 +38,11 @@ import java.io.IOException;
import java.sql.Array; import java.sql.Array;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.*; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Predicate; import java.util.function.Predicate;
@ -174,7 +178,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
execute(sqlFile, producer, oaf -> true); execute(sqlFile, producer, oaf -> true);
} }
public void execute(final String sqlFile, final Function<ResultSet, List<Oaf>> producer, public void execute(final String sqlFile,
final Function<ResultSet, List<Oaf>> producer,
final Predicate<Oaf> predicate) final Predicate<Oaf> predicate)
throws Exception { throws Exception {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile)); final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/sql/" + sqlFile));
@ -198,8 +203,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
ds ds
.setOriginalId( .setOriginalId(
Arrays Arrays
.asList( .asList((String[]) rs.getArray("identities").getArray())
(String[]) rs.getArray("identities").getArray())
.stream() .stream()
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.collect(Collectors.toList())); .collect(Collectors.toList()));
@ -250,11 +254,8 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
ds ds
.setJournal( .setJournal(
journal( journal(
rs.getString("officialname"), rs.getString("officialname"), rs.getString("issnPrinted"), rs.getString("issnOnline"),
rs.getString("issnPrinted"), rs.getString("issnLinking"), info)); // Journal
rs.getString("issnOnline"),
rs.getString("issnLinking"),
info)); // Journal
ds.setDataInfo(info); ds.setDataInfo(info);
ds.setLastupdatetimestamp(lastUpdateTimestamp); ds.setLastupdatetimestamp(lastUpdateTimestamp);
@ -332,7 +333,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
listKeyValues( listKeyValues(
createOpenaireId(10, rs.getString("collectedfromid"), true), createOpenaireId(10, rs.getString("collectedfromid"), true),
rs.getString("collectedfromname"))); rs.getString("collectedfromname")));
o.setPid(new ArrayList<>()); o.setPid(prepareListOfStructProps(rs.getArray("pid"), info));
o.setDateofcollection(asString(rs.getDate("dateofcollection"))); o.setDateofcollection(asString(rs.getDate("dateofcollection")));
o.setDateoftransformation(asString(rs.getDate("dateoftransformation"))); o.setDateoftransformation(asString(rs.getDate("dateoftransformation")));
o.setExtraInfo(new ArrayList<>()); // Values not present in the DB o.setExtraInfo(new ArrayList<>()); // Values not present in the DB

32
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql
View File

@ -24,12 +24,30 @@ SELECT
d.officialname AS collectedfromname, d.officialname AS collectedfromname,
o.country || '@@@dnet:countries' AS country, o.country || '@@@dnet:countries' AS country,
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
ARRAY[]::text[] AS pid array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype), NULL) AS pid
FROM dsm_organizations o FROM dsm_organizations o
LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom) LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom)
LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id)
LEFT OUTER JOIN dsm_identities i ON (i.pid = p.pid)
GROUP BY
o.id,
o.legalshortname,
o.legalname,
o.websiteurl,
o.logourl,
o.ec_legalbody,
o.ec_legalperson,
o.ec_nonprofit,
o.ec_researchorganization,
o.ec_highereducation,
o.ec_internationalorganizationeurinterests,
o.ec_internationalorganization,
o.ec_enterprise,
o.ec_smevalidated,
o.ec_nutscode,
o.dateofcollection,
o.lastupdate,
o.trust,
d.id,
d.officialname,
o.country