D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

Restructure workflow parameters

This commit is contained in:
Serafeim Chatzopoulos 2023-10-03 15:11:58 +03:00
parent 9f73d93e62
commit 24c43e0c60
3 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties
View File

@ -1,8 +1,6 @@
# hive # hive
hiveDbName=openaire_prod_20230914 hiveDbName=openaire_prod_20230914
sparkSqlWarehouseDir=/user/hive/warehouse
# input/output files # input/output files
softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv
lastVisitsPath=${workingDir}/2_last_visits.seq lastVisitsPath=${workingDir}/2_last_visits.seq

4
dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml
View File

@ -47,4 +47,8 @@
<name>oozie.launcher.mapreduce.user.classpath.first</name> <name>oozie.launcher.mapreduce.user.classpath.first</name>
<value>true</value> <value>true</value>
</property> </property>
<property>
<name>sparkSqlWarehouseDir</name>
<value>/user/hive/warehouse</value>
</property>
</configuration> </configuration>

12
dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml
View File

@ -18,6 +18,14 @@
<name>archiveRequestsPath</name> <name>archiveRequestsPath</name>
<description>The path in the HDFS to save the responses of the archive requests</description> <description>The path in the HDFS to save the responses of the archive requests</description>
</property> </property>
<property>
<name>actionsetsPath</name>
<description>The path in the HDFS to save the action sets</description>
</property>
<property>
<name>graphPath</name>
<description>The path in the HDFS to the base folder of the graph</description>
</property>
<property> <property>
<name>maxNumberOfRetry</name> <name>maxNumberOfRetry</name>
<description>Max number of retries for failed API calls</description> <description>Max number of retries for failed API calls</description>
@ -30,6 +38,10 @@
<name>requestDelay</name> <name>requestDelay</name>
<description>Delay between API requests (in ms)</description> <description>Delay between API requests (in ms)</description>
</property> </property>
<property>
<name>softwareLimit</name>
<description>Limit on the number of repo URLs to use (Optional); for debug purposes</description>
</property>
<property> <property>
<name>resume</name> <name>resume</name>
<description>Variable that indicates the step to start from</description> <description>Variable that indicates the step to start from</description>