Restructure workflow parameters

This commit is contained in:
Serafeim Chatzopoulos 2023-10-03 15:11:58 +03:00
parent 9f73d93e62
commit 24c43e0c60
3 changed files with 16 additions and 2 deletions

View File

@ -1,8 +1,6 @@
# hive
hiveDbName=openaire_prod_20230914
sparkSqlWarehouseDir=/user/hive/warehouse
# input/output files
softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv
lastVisitsPath=${workingDir}/2_last_visits.seq

View File

@ -47,4 +47,8 @@
<name>oozie.launcher.mapreduce.user.classpath.first</name>
<value>true</value>
</property>
<property>
<name>sparkSqlWarehouseDir</name>
<value>/user/hive/warehouse</value>
</property>
</configuration>

View File

@ -18,6 +18,14 @@
<name>archiveRequestsPath</name>
<description>The path in the HDFS to save the responses of the archive requests</description>
</property>
<property>
<name>actionsetsPath</name>
<description>The path in the HDFS to save the action sets</description>
</property>
<property>
<name>graphPath</name>
<description>The path in the HDFS to the base folder of the graph</description>
</property>
<property>
<name>maxNumberOfRetry</name>
<description>Max number of retries for failed API calls</description>
@ -30,6 +38,10 @@
<name>requestDelay</name>
<description>Delay between API requests (in ms)</description>
</property>
<property>
<name>softwareLimit</name>
<description>Limit on the number of repo URLs to use (Optional); for debug purposes</description>
</property>
<property>
<name>resume</name>
<description>Variable that indicates the step to start from</description>