From c2a6e2a9bf45907905ac45e0ed5a0ace4622e71b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 2 Oct 2020 09:37:08 +0200 Subject: [PATCH] fixed mapping for datasource journal info (ISSNs) --- .../raw/MigrateDbEntitiesApplication.java | 39 +++++++++---------- .../oa/graph/raw/common/OafMapperUtils.java | 21 ++++++++++ .../dhp/oa/graph/sql/queryDatasources.sql | 6 ++- .../raw/MigrateDbEntitiesApplicationTest.java | 6 +-- .../raw/datasources_resultset_entry.json | 14 ++++++- 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java index 1e7b56ee9c..a0340a2b4d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java @@ -38,13 +38,11 @@ import java.io.IOException; import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -197,7 +195,11 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i final Datasource ds = new Datasource(); ds.setId(createOpenaireId(10, rs.getString("datasourceid"), true)); - ds.setOriginalId(Arrays.asList((String[]) rs.getArray("identities").getArray())); + ds.setOriginalId(Arrays.asList( + (String[]) rs.getArray("identities").getArray()) + .stream() + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList())); ds .setCollectedfrom( listKeyValues( @@ -242,8 +244,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i ds.setPidsystems(field(rs.getString("pidsystems"), info)); ds.setCertificates(field(rs.getString("certificates"), info)); ds.setPolicies(new ArrayList<>()); // The sql query returns an empty array - ds - .setJournal(prepareJournal(rs.getString("officialname"), rs.getString("journal"), info)); // Journal + ds.setJournal( + journal( + rs.getString("officialname"), + rs.getString("issnPrinted"), + rs.getString("issnOnline"), + rs.getString("issnLinking"), + info)); // Journal ds.setDataInfo(info); ds.setLastupdatetimestamp(lastUpdateTimestamp); @@ -567,21 +574,13 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i return res; } - private Journal prepareJournal(final String name, final String sj, final DataInfo info) { - if (StringUtils.isNotBlank(sj)) { - final String[] arr = sj.split("@@@"); - if (arr.length == 3) { - final String issn = StringUtils.isNotBlank(arr[0]) ? arr[0].trim() : null; - final String eissn = StringUtils.isNotBlank(arr[1]) ? arr[1].trim() : null; + private Journal prepareJournal(final ResultSet rs, final DataInfo info) throws SQLException { + if (Objects.isNull(rs)) { + return null; + } else { - final String lissn = StringUtils.isNotBlank(arr[2]) ? arr[2].trim() : null; - - if (issn != null || eissn != null || lissn != null) { - return journal(name, issn, eissn, lissn, null, null, null, null, null, null, null, info); - } - } + return journal(rs.getString("officialname"), rs.getString("issnPrinted"), rs.getString("issnOnline"), rs.getString("issnLinking"), info); } - return null; } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java index 63db13b8fb..7c2659359f 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/common/OafMapperUtils.java @@ -153,6 +153,27 @@ public class OafMapperUtils { return p; } + public static Journal journal( + final String name, + final String issnPrinted, + final String issnOnline, + final String issnLinking, + final DataInfo dataInfo) { + return journal( + name, + issnPrinted, + issnOnline, + issnLinking, + null, + null, + null, + null, + null, + null, + null, + dataInfo); + } + public static Journal journal( final String name, final String issnPrinted, diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql index 7ca672835b..f0a4161ab9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql @@ -84,8 +84,10 @@ SELECT dc.id AS collectedfromid, dc.officialname AS collectedfromname, d.typology||'@@@dnet:datasource_typologies' AS datasourcetype, - 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, - concat_ws(' @@@ ', d.issn, d.eissn, d.lissn) AS journal + 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, + d.issn AS issnPrinted, + d.eissn AS issnOnline, + d.lissn AS issnLinking FROM dsm_datasources d diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java index 011cc18e6e..f663d60957 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java @@ -80,9 +80,9 @@ public class MigrateDbEntitiesApplicationTest { assertEquals(getValueAsString("namespaceprefix", fields), ds.getNamespaceprefix().getValue()); assertEquals(getValueAsString("collectedfromname", fields), ds.getCollectedfrom().get(0).getValue()); assertEquals(getValueAsString("officialname", fields), ds.getJournal().getName()); - assertEquals("2579-5449", ds.getJournal().getIssnPrinted()); - assertEquals("2597-6540", ds.getJournal().getIssnOnline()); - assertEquals(null, ds.getJournal().getIssnLinking()); + assertEquals(getValueAsString("issnPrinted", fields), ds.getJournal().getIssnPrinted()); + assertEquals(getValueAsString("issnOnline", fields), ds.getJournal().getIssnOnline()); + assertEquals(getValueAsString("issnLinking", fields), ds.getJournal().getIssnLinking()); } @Test diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json index 0f1da70959..8f8aed3a0e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json @@ -228,8 +228,18 @@ "value": "sysimport:crosswalk:entityregistry@@@dnet:provenance_actions" }, { - "field": "journal", + "field": "issnPrinted", "type": "string", - "value": "2579-5449 @@@ 2597-6540 @@@ " + "value": "2579-5449" + }, + { + "field": "issnOnline", + "type": "string", + "value": "2579-5448" + }, + { + "field": "issnLinking", + "type": "string", + "value": "2579-5447" } ]