D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

addition of alternative names in export queries

This commit is contained in:
miconis 2021-04-20 12:45:21 +02:00
parent cadd0a5de8
commit 0393cdce42
6 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java
View File

@ -354,7 +354,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
o.setOaiprovenance(null); // Values not present in the DB o.setOaiprovenance(null); // Values not present in the DB
o.setLegalshortname(field(rs.getString("legalshortname"), info)); o.setLegalshortname(field(rs.getString("legalshortname"), info));
o.setLegalname(field(rs.getString("legalname"), info)); o.setLegalname(field(rs.getString("legalname"), info));
o.setAlternativeNames(new ArrayList<>()); // Values not returned by the SQL query o.setAlternativeNames(prepareListFields(rs.getArray("alternativenames"), info));
o.setWebsiteurl(field(rs.getString("websiteurl"), info)); o.setWebsiteurl(field(rs.getString("websiteurl"), info));
o.setLogourl(field(rs.getString("logourl"), info)); o.setLogourl(field(rs.getString("logourl"), info));
o.setEclegalbody(field(Boolean.toString(rs.getBoolean("eclegalbody")), info)); o.setEclegalbody(field(Boolean.toString(rs.getBoolean("eclegalbody")), info));

4
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForOrgsDedup.sql
View File

@ -2,7 +2,7 @@ SELECT
o.id AS organizationid, o.id AS organizationid,
coalesce((array_agg(a.acronym))[1], o.name) AS legalshortname, coalesce((array_agg(a.acronym))[1], o.name) AS legalshortname,
o.name AS legalname, o.name AS legalname,
array_agg(DISTINCT n.name) AS "alternativeNames", array_remove(array_cat(array_agg(DISTINCT n.name), array_agg(DISTINCT a.acronym)), NULL) AS alternativenames,
(array_agg(u.url))[1] AS websiteurl, (array_agg(u.url))[1] AS websiteurl,
'' AS logourl, '' AS logourl,
o.creation_date AS dateofcollection, o.creation_date AS dateofcollection,
@ -46,7 +46,7 @@ SELECT
'openorgsmesh'||substring(o.id, 13)||'-'||md5(n.name) AS organizationid, 'openorgsmesh'||substring(o.id, 13)||'-'||md5(n.name) AS organizationid,
n.name AS legalshortname, n.name AS legalshortname,
n.name AS legalname, n.name AS legalname,
ARRAY[]::text[] AS "alternativeNames", ARRAY[]::text[] AS alternativenames,
(array_agg(u.url))[1] AS websiteurl, (array_agg(u.url))[1] AS websiteurl,
'' AS logourl, '' AS logourl,
o.creation_date AS dateofcollection, o.creation_date AS dateofcollection,

2
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql
View File

@ -2,7 +2,7 @@ SELECT
o.id AS organizationid, o.id AS organizationid,
coalesce((array_agg(a.acronym))[1], o.name) AS legalshortname, coalesce((array_agg(a.acronym))[1], o.name) AS legalshortname,
o.name AS legalname, o.name AS legalname,
array_agg(DISTINCT n.name) AS "alternativeNames", array_remove(array_cat(array_agg(DISTINCT n.name), array_agg(DISTINCT a.acronym)), NULL) AS alternativenames,
(array_agg(u.url))[1] AS websiteurl, (array_agg(u.url))[1] AS websiteurl,
'' AS logourl, '' AS logourl,
o.creation_date AS dateofcollection, o.creation_date AS dateofcollection,

3
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOrganizations.sql
View File

@ -23,8 +23,9 @@ SELECT
d.id AS collectedfromid, d.id AS collectedfromid,
d.officialname AS collectedfromname, d.officialname AS collectedfromname,
o.country || '@@@dnet:countries' AS country, o.country || '@@@dnet:countries' AS country,
array[]::text[] AS alternativenames,
'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction, 'sysimport:crosswalk:entityregistry@@@dnet:provenance_actions' AS provenanceaction,
array_agg(DISTINCT i.pid || '###' || i.issuertype || '@@@dnet:pid_types') AS pid array_agg(DISTINCT i.pid || '###' || i.issuertype || '@@@dnet:pid_types') AS pid
FROM dsm_organizations o FROM dsm_organizations o
LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom) LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom)
LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id) LEFT OUTER JOIN dsm_organizationpids p ON (p.organization = o.id)

9
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java
View File

@ -15,6 +15,7 @@ import java.sql.SQLException;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
@ -118,6 +119,10 @@ public class MigrateDbEntitiesApplicationTest {
assertEquals(getValueAsString("country", fields).split("@@@")[1], o.getCountry().getSchemeid()); assertEquals(getValueAsString("country", fields).split("@@@")[1], o.getCountry().getSchemeid());
assertEquals(getValueAsString("country", fields).split("@@@")[1], o.getCountry().getSchemename()); assertEquals(getValueAsString("country", fields).split("@@@")[1], o.getCountry().getSchemename());
assertEquals(getValueAsString("collectedfromname", fields), o.getCollectedfrom().get(0).getValue()); assertEquals(getValueAsString("collectedfromname", fields), o.getCollectedfrom().get(0).getValue());
List<String> alternativenames = getValueAsList("alternativenames", fields);
assertEquals(2, alternativenames.size());
assertTrue(alternativenames.contains("Pippo"));
assertTrue(alternativenames.contains("Foo"));
} }
@Test @Test
@ -339,6 +344,10 @@ public class MigrateDbEntitiesApplicationTest {
.findFirst() .findFirst()
.get(); .get();
} }
private List<String> getValueAsList(final String name, final List<TypedField> fields) {
return (List<String>) getValueAs(name, fields);
}
} }
class TypedField { class TypedField {

7
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/organizations_resultset_entry.json
View File

@ -123,5 +123,12 @@
"field": "provenanceaction", "field": "provenanceaction",
"type": "not_used", "type": "not_used",
"value": "sysimport:crosswalk:entityregistry@@@dnet:provenance_actions" "value": "sysimport:crosswalk:entityregistry@@@dnet:provenance_actions"
},
{
"field": "alternativenames",
"type": "array",
"value": [
"Pippo", "Foo"
]
} }
] ]