Merge branch 'beta' into instance_group_by_url

dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java

@@ -107,7 +107,7 @@ public class DedupRecordFactory {
 		entity.setId(id);
 
 		entity.setLastupdatetimestamp(ts);
-		entity.getDataInfo().setDeletedbyinference(false);
+		entity.setDataInfo(dataInfo);
 
 		return entity;
 	}

dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java

@@ -77,7 +77,16 @@ class EntityMergerTest implements Serializable {
 		// verify id
 		assertEquals(dedupId, pub_merged.getId());
 
-		assertEquals(pub_top.getJournal(), pub_merged.getJournal());
+		assertEquals(pub_top.getJournal().getName(), pub_merged.getJournal().getName());
+		assertEquals(pub_top.getJournal().getIssnOnline(), pub_merged.getJournal().getIssnOnline());
+		assertEquals(pub_top.getJournal().getIssnLinking(), pub_merged.getJournal().getIssnLinking());
+		assertEquals(pub_top.getJournal().getIssnPrinted(), pub_merged.getJournal().getIssnPrinted());
+		assertEquals(pub_top.getJournal().getIss(), pub_merged.getJournal().getIss());
+		assertEquals(pub_top.getJournal().getEp(), pub_merged.getJournal().getEp());
+		assertEquals(pub_top.getJournal().getSp(), pub_merged.getJournal().getSp());
+		assertEquals(pub_top.getJournal().getVol(), pub_merged.getJournal().getVol());
+		assertEquals(pub_top.getJournal().getConferencedate(), pub_merged.getJournal().getConferencedate());
+		assertEquals(pub_top.getJournal().getConferenceplace(), pub_merged.getJournal().getConferenceplace());
 		assertEquals("OPEN", pub_merged.getBestaccessright().getClassid());
 		assertEquals(pub_top.getResulttype(), pub_merged.getResulttype());
 		assertEquals(pub_top.getLanguage(), pub_merged.getLanguage());

dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/countrypropagation/input_countrypropagation_parameters.json

@@ -5,12 +5,6 @@
     "paramDescription": "the path of the sequencial file to read",
     "paramRequired": true
   },
-  {
-    "paramName":"h",
-    "paramLongName":"hive_metastore_uris",
-    "paramDescription": "the hive metastore uris",
-    "paramRequired": false
-  },
   {
     "paramName":"tn",
     "paramLongName":"resultTableName",

dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/countrypropagation/oozie_app/workflow.xml

@@ -110,7 +110,6 @@
             <arg>--sourcePath</arg><arg>${sourcePath}</arg>
             <arg>--whitelist</arg><arg>${whitelist}</arg>
             <arg>--allowedtypes</arg><arg>${allowedtypes}</arg>
-            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
             <arg>--outputPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
         <ok to="fork_join_prepare_result_country"/>
@@ -146,7 +145,7 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
             <arg>--outputPath</arg><arg>${workingDir}/publication</arg>
-            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
+            <arg>--workingPath</arg><arg>${workingDir}/workingP</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
@@ -176,7 +175,7 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
             <arg>--outputPath</arg><arg>${workingDir}/dataset</arg>
-            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
+            <arg>--workingPath</arg><arg>${workingDir}/workingD</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
@@ -206,7 +205,7 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
             <arg>--outputPath</arg><arg>${workingDir}/otherresearchproduct</arg>
-            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
+            <arg>--workingPath</arg><arg>${workingDir}/workingO</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>
@@ -236,7 +235,7 @@
             </spark-opts>
             <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
             <arg>--outputPath</arg><arg>${workingDir}/software</arg>
-            <arg>--hive_metastore_uris</arg><arg>${hive_metastore_uris}</arg>
+            <arg>--workingPath</arg><arg>${workingDir}/workingS</arg>
             <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
             <arg>--preparedInfoPath</arg><arg>${workingDir}/preparedInfo</arg>
         </spark>

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java

@@ -309,6 +309,9 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i
 			ds.setThematic(rs.getBoolean("thematic"));
 			ds.setKnowledgegraph(rs.getBoolean("knowledgegraph"));
 			ds.setContentpolicies(prepareListOfQualifiers(rs.getArray("contentpolicies")));
+			ds.setConsenttermsofuse(rs.getBoolean("consenttermsofuse"));
+			ds.setFulltextdownload(rs.getBoolean("fulltextdownload"));
+			ds.setConsenttermsofusedate(rs.getDate("consenttermsofusedate").toString());
 
 			return Arrays.asList(ds);
 		} catch (final Exception e) {

dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryDatasources.sql

@@ -89,6 +89,9 @@ SELECT
 	d.issn                                                                                                     AS issnPrinted,
 	d.eissn                                                                                                    AS issnOnline,
 	d.lissn                                                                                                    AS issnLinking,
+	d.consenttermsofuse                                                                                        AS consenttermsofuse,
+	d.fulltextdownload                                                                                         AS fulltextdownload,
+	d.consenttermsofusedate                                                                                    AS consenttermsofusedate,
 	de.jurisdiction||'@@@eosc:jurisdictions'                                                                   AS jurisdiction,
 	de.thematic                                                                                                AS thematic,
 	de.knowledge_graph                                                                                         AS knowledgegraph,

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplicationTest.java

@@ -100,6 +100,9 @@ public class MigrateDbEntitiesApplicationTest {
 		assertEquals("Journal article", ds.getContentpolicies().get(0).getClassid());
 		assertEquals("eosc:contentpolicies", ds.getContentpolicies().get(0).getSchemeid());
 
+		assertEquals(true, ds.getConsenttermsofuse());
+		assertEquals(true, ds.getFulltextdownload());
+		assertEquals("2022-03-11", ds.getConsenttermsofusedate());
 	}
 
 	@Test

dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/datasources_resultset_entry.json

@@ -268,5 +268,20 @@
 		"value": [
 			"Journal article@@@eosc:contentpolicies"
 		]
+	},
+	{
+		"field": "consenttermsofuse",
+		"type": "boolean",
+		"value": true
+	},
+	{
+		"field": "fulltextdownload",
+		"type": "boolean",
+		"value": true
+	},
+	{
+		"field": "consenttermsofusedate",
+		"type": "date",
+		"value": "2022-03-11"
 	}
 ]