antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'beta' into instance_group_by_url

This commit is contained in:
Claudio Atzori 2022-03-28 09:23:52 +02:00
parent 395ac6ecec fa3cb84f77
commit 96aa2a5d0d
8 changed files with 39 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java
View File

@ -107,7 +107,7 @@ public class DedupRecordFactory {
entity.setId(id);
entity.setLastupdatetimestamp(ts);
entity.getDataInfo().setDeletedbyinference(false);
entity.setDataInfo(dataInfo);
return entity;
}

11
dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java
View File

@ -77,7 +77,16 @@ class EntityMergerTest implements Serializable {
// verify id
assertEquals(dedupId, pub_merged.getId());
assertEquals(pub_top.getJournal(), pub_merged.getJournal());
assertEquals(pub_top.getJournal().getName(), pub_merged.getJournal().getName());
assertEquals(pub_top.getJournal().getIssnOnline(), pub_merged.getJournal().getIssnOnline());
assertEquals(pub_top.getJournal().getIssnLinking(), pub_merged.getJournal().getIssnLinking());
assertEquals(pub_top.getJournal().getIssnPrinted(), pub_merged.getJournal().getIssnPrinted());
assertEquals(pub_top.getJournal().getIss(), pub_merged.getJournal().getIss());
assertEquals(pub_top.getJournal().getEp(), pub_merged.getJournal().getEp());
assertEquals(pub_top.getJournal().getSp(), pub_merged.getJournal().getSp());
assertEquals(pub_top.getJournal().getVol(), pub_merged.getJournal().getVol());
assertEquals(pub_top.getJournal().getConferencedate(), pub_merged.getJournal().getConferencedate());
assertEquals(pub_top.getJournal().getConferenceplace(), pub_merged.getJournal().getConferenceplace());
assertEquals("OPEN", pub_merged.getBestaccessright().getClassid());
assertEquals(pub_top.getResulttype(), pub_merged.getResulttype());
assertEquals(pub_top.getLanguage(), pub_merged.getLanguage());

6
dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/countrypropagation/input_countrypropagation_parameters.json
View File

@ -5,12 +5,6 @@
"paramDescription": "the path of the sequencial file to read",
"paramRequired": true
},
{
"paramName":"h",
"paramLongName":"hive_metastore_uris",
"paramDescription": "the hive metastore uris",
"paramRequired": false
},
{
"paramName":"tn",
"paramLongName":"resultTableName",

9
dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/countrypropagation/oozie_app/workflow.xml
View File

@ -110,7 +110,6 @@
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
<arg>--whitelist</arg><arg>${whitelist}</arg>
<arg>--allowedtypes</arg><arg>${allowedtypes}</arg>
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
<arg>--outputPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark>
<ok to="fork_join_prepare_result_country"/>
@ -146,7 +145,7 @@
</spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
<arg>--outputPath</arg><arg>${workingDir}/publication</arg>
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
<arg>--workingPath</arg><arg>${workingDir}/workingP</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark>
@ -176,7 +175,7 @@
</spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
<arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
<arg>--workingPath</arg><arg>${workingDir}/workingD</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark>
@ -206,7 +205,7 @@
</spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
<arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
<arg>--workingPath</arg><arg>${workingDir}/workingO</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark>
@ -236,7 +235,7 @@
</spark-opts>
<arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
<arg>--outputPath</arg><arg>${workingDir}/software</arg>
<arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
<arg>--workingPath</arg><arg>${workingDir}/workingS</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
</spark>

3
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java
View File

@ -309,6 +309,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
ds.setThematic(rs.getBoolean("thematic"));
ds.setKnowledgegraph(rs.getBoolean("knowledgegraph"));
ds.setContentpolicies(prepareListOfQualifiers(rs.getArray("contentpolicies")));
ds.setConsenttermsofuse(rs.getBoolean("consenttermsofuse"));
ds.setFulltextdownload(rs.getBoolean("fulltextdownload"));
ds.setConsenttermsofusedate(rs.getDate("consenttermsofusedate").toString());
return Arrays.asList(ds);
} catch (final Exception e) {

3
dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql
View File

@ -89,6 +89,9 @@ SELECT
d.issn AS issnPrinted,
d.eissn AS issnOnline,
d.lissn AS issnLinking,
d.consenttermsofuse AS consenttermsofuse,
d.fulltextdownload AS fulltextdownload,
d.consenttermsofusedate AS consenttermsofusedate,
de.jurisdiction||'@@@eosc:jurisdictions' AS jurisdiction,
de.thematic AS thematic,
de.knowledge_graph AS knowledgegraph,

3
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java
View File

@ -100,6 +100,9 @@ public class MigrateDbEntitiesApplicationTest {
assertEquals("Journal article", ds.getContentpolicies().get(0).getClassid());
assertEquals("eosc:contentpolicies", ds.getContentpolicies().get(0).getSchemeid());
assertEquals(true, ds.getConsenttermsofuse());
assertEquals(true, ds.getFulltextdownload());
assertEquals("2022-03-11", ds.getConsenttermsofusedate());
}
@Test

15
dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json
View File

@ -268,5 +268,20 @@
"value": [
"Journal article@@@eosc:contentpolicies"
]
},
{
"field": "consenttermsofuse",
"type": "boolean",
"value": true
},
{
"field": "fulltextdownload",
"type": "boolean",
"value": true
},
{
"field": "consenttermsofusedate",
"type": "date",
"value": "2022-03-11"
}
]