From 64bc955444d2acd8fc9afbec3224ff9305b9a287 Mon Sep 17 00:00:00 2001
From: Enrico Ottonello <enrico.ottonello@isti.cnr.it>
Date: Tue, 19 Jul 2022 12:12:01 +0200
Subject: [PATCH] new script to nquads download from scraping service

---
 .../dhp/bioschema/oozie_app/workflow.xml      |  4 ++--
 .../bioschema/oozie_app/download_nquads.sh    |  3 +++
 .../bioschema/oozie_app/workflow.xml          | 20 ++++++++++++++++++-
 3 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100644 dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/download_nquads.sh

diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/bioschema/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/bioschema/oozie_app/workflow.xml
index 43f39f6c2..deacb878b 100644
--- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/bioschema/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/bioschema/oozie_app/workflow.xml
@@ -2,12 +2,12 @@
     <parameters>
         <property>
             <name>mainPath</name>
-            <value>/data/bioschema/ped</value>
+            <value>/data/bioschema/mobidb</value>
             <description>the working path of Bioschema stores</description>
         </property>
         <property>
             <name>datasourceKey</name>
-            <value>ped</value>
+            <value>mobidb</value>
             <description>the key that identifies the datasource (eg ped, disprot, mobidb)</description>
         </property>
         <property>
diff --git a/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/download_nquads.sh b/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/download_nquads.sh
new file mode 100644
index 000000000..019241fa7
--- /dev/null
+++ b/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/download_nquads.sh
@@ -0,0 +1,3 @@
+wget -O /tmp/base64_gzipped_nquads.txt https://hadoop-bioschemas-ds.garr-pa1.d4science.org/bioschemas-api/api/getNQuads?datasourceKey=$1
+hdfs dfs -copyFromLocal /tmp/base64_gzipped_nquads.txt /data/bioschema/mobidb
+rm -f /tmp/base64_gzipped_nquads.txt
\ No newline at end of file
diff --git a/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/workflow.xml b/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/workflow.xml
index 821fbf9bd..7305a2740 100644
--- a/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-rdfconverter/src/main/resources/eu/dnetlib/dhp/rdfconverter/bioschema/oozie_app/workflow.xml
@@ -1,5 +1,10 @@
 <workflow-app name="RdfConverter" xmlns="uri:oozie:workflow:0.5">
     <parameters>
+        <property>
+            <name>bioschemas_datasource_key</name>
+            <value>mobidb</value>
+            <description>bioschemas datasource key (i.e. mobidb, ped, disprot)</description>
+        </property>
         <property>
             <name>workingPath</name>
             <value>/data/bioschema/mobidb/</value>
@@ -57,11 +62,24 @@
         </property>
     </parameters>
 
-    <start to="ResetWorkingPath"/>
+    <start to="DownloadNQuads"/>
     <kill name="Kill">
         <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
     </kill>
 
+    <action name="DownloadNQuads">
+        <shell xmlns="uri:oozie:shell-action:0.1">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <exec>download_nquads.sh</exec>
+            <argument>${bioschemas_datasource_key}</argument>
+            <file>download_nquads.sh</file>
+            <capture-output/>
+        </shell>
+        <ok to="ResetWorkingPath"/>
+        <error to="Kill"/>
+    </action>
+
     <action name="ResetWorkingPath">
         <fs>
             <delete path='${workingPath}${output}'/>