dimitris.pierrakos
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Add actionsetsPath as a global WF param

This commit is contained in:
Serafeim Chatzopoulos 2023-10-03 15:43:38 +03:00
parent 24c43e0c60
commit b49a3ac9b2
2 changed files with 16 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java
View File

@ -1,12 +1,12 @@
package eu.dnetlib.dhp.swh; package eu.dnetlib.dhp.swh;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration;
import eu.dnetlib.dhp.common.collection.CollectorException;
import eu.dnetlib.dhp.common.collection.HttpClientParams; import java.io.BufferedReader;
import eu.dnetlib.dhp.swh.utils.SWHConnection; import java.io.IOException;
import eu.dnetlib.dhp.swh.utils.SWHConstants; import java.net.URL;
import eu.dnetlib.dhp.swh.utils.SWHUtils;
import org.apache.commons.cli.ParseException; import org.apache.commons.cli.ParseException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
@ -16,11 +16,12 @@ import org.apache.hadoop.io.SequenceFile;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.BufferedReader; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import java.io.IOException; import eu.dnetlib.dhp.common.collection.CollectorException;
import java.net.URL; import eu.dnetlib.dhp.common.collection.HttpClientParams;
import eu.dnetlib.dhp.swh.utils.SWHConnection;
import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; import eu.dnetlib.dhp.swh.utils.SWHConstants;
import eu.dnetlib.dhp.swh.utils.SWHUtils;
/** /**
* Given a file with software repository URLs, this class * Given a file with software repository URLs, this class

4
dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml
View File

@ -57,6 +57,10 @@
<name>oozie.action.sharelib.for.spark</name> <name>oozie.action.sharelib.for.spark</name>
<value>${oozieActionShareLibForSpark2}</value> <value>${oozieActionShareLibForSpark2}</value>
</property> </property>
<property>
<name>actionsetsPath</name>
<value>${actionsetsPath}</value>
</property>
</configuration> </configuration>
</global> </global>