lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

changed the name of some parameters

This commit is contained in:
Miriam Baglioni 2020-05-04 13:58:52 +02:00
parent b7dd400e51
commit 3957c815b9
4 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-bulktag/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob2.java
View File

@ -57,9 +57,9 @@ public class SparkBulkTagJob2 {
final String outputPath = parser.get("outputPath"); final String outputPath = parser.get("outputPath");
log.info("outputPath: {}", outputPath); log.info("outputPath: {}", outputPath);
ProtoMap protoMappingParams = new Gson().fromJson(parser.get("protoMap"), ProtoMap.class); ProtoMap protoMappingParams = new Gson().fromJson(parser.get("pathMap"), ProtoMap.class);
; ;
log.info("protoMap: {}", new Gson().toJson(protoMappingParams)); log.info("pathMap: {}", new Gson().toJson(protoMappingParams));
final String resultClassName = parser.get("resultTableName"); final String resultClassName = parser.get("resultTableName");
log.info("resultTableName: {}", resultClassName); log.info("resultTableName: {}", resultClassName);
@ -80,7 +80,7 @@ public class SparkBulkTagJob2 {
if (isTest) { if (isTest) {
cc = CommunityConfigurationFactory.newInstance(taggingConf); cc = CommunityConfigurationFactory.newInstance(taggingConf);
} else { } else {
cc = QueryInformationSystem.getCommunityConfiguration(parser.get("isLookupUrl")); cc = QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl"));
} }
runWithSparkSession( runWithSparkSession(

4
dhp-workflows/dhp-bulktag/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json
View File

@ -1,7 +1,7 @@
[ [
{ {
"paramName":"is", "paramName":"is",
"paramLongName":"isLookupUrl", "paramLongName":"isLookUpUrl",
"paramDescription": "URL of the isLookUp Service", "paramDescription": "URL of the isLookUp Service",
"paramRequired": true "paramRequired": true
}, },
@ -13,7 +13,7 @@
}, },
{ {
"paramName": "pm", "paramName": "pm",
"paramLongName":"protoMap", "paramLongName":"pathMap",
"paramDescription": "the json path associated to each selection field", "paramDescription": "the json path associated to each selection field",
"paramRequired": true "paramRequired": true
}, },

20
dhp-workflows/dhp-bulktag/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml
View File

@ -5,11 +5,11 @@
<description>the source path</description> <description>the source path</description>
</property> </property>
<property> <property>
<name>isLookupUrl</name> <name>isLookUpUrl</name>
<description>the isLookup service endpoint</description> <description>the isLookup service endpoint</description>
</property> </property>
<property> <property>
<name>protoMap</name> <name>pathMap</name>
<description>the json path associated to each selection field</description> <description>the json path associated to each selection field</description>
</property> </property>
<property> <property>
@ -121,8 +121,8 @@
<arg>--sourcePath</arg><arg>${sourcePath}/publication</arg> <arg>--sourcePath</arg><arg>${sourcePath}/publication</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Publication</arg>
<arg>--outputPath</arg><arg>${outputPath}/publication</arg> <arg>--outputPath</arg><arg>${outputPath}/publication</arg>
<arg>--protoMap</arg><arg>${protoMap}</arg> <arg>--pathMap</arg><arg>${pathMap}</arg>
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
<ok to="wait"/> <ok to="wait"/>
<error to="Kill"/> <error to="Kill"/>
@ -149,8 +149,8 @@
<arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg> <arg>--sourcePath</arg><arg>${sourcePath}/dataset</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Dataset</arg>
<arg>--outputPath</arg><arg>${outputPath}/dataset</arg> <arg>--outputPath</arg><arg>${outputPath}/dataset</arg>
<arg>--protoMap</arg><arg>${protoMap}</arg> <arg>--pathMap</arg><arg>${pathMap}</arg>
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
<ok to="wait"/> <ok to="wait"/>
<error to="Kill"/> <error to="Kill"/>
@ -177,8 +177,8 @@
<arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg> <arg>--sourcePath</arg><arg>${sourcePath}/otherresearchproduct</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.OtherResearchProduct</arg>
<arg>--outputPath</arg><arg>${outputPath}/otherresearchproduct</arg> <arg>--outputPath</arg><arg>${outputPath}/otherresearchproduct</arg>
<arg>--protoMap</arg><arg>${protoMap}</arg> <arg>--pathMap</arg><arg>${pathMap}</arg>
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
<ok to="wait"/> <ok to="wait"/>
<error to="Kill"/> <error to="Kill"/>
@ -205,8 +205,8 @@
<arg>--sourcePath</arg><arg>${sourcePath}/software</arg> <arg>--sourcePath</arg><arg>${sourcePath}/software</arg>
<arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg> <arg>--resultTableName</arg><arg>eu.dnetlib.dhp.schema.oaf.Software</arg>
<arg>--outputPath</arg><arg>${outputPath}/software</arg> <arg>--outputPath</arg><arg>${outputPath}/software</arg>
<arg>--protoMap</arg><arg>${protoMap}</arg> <arg>--pathMap</arg><arg>${pathMap}</arg>
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg> <arg>--isLookUpUrl</arg><arg>${isLookUpUrl}</arg>
</spark> </spark>
<ok to="wait"/> <ok to="wait"/>
<error to="Kill"/> <error to="Kill"/>

32
dhp-workflows/dhp-bulktag/src/test/java/eu/dnetlib/dhp/BulkTagJobTest.java
View File

@ -99,9 +99,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Dataset", "eu.dnetlib.dhp.schema.oaf.Dataset",
"-outputPath", "-outputPath",
workingDir.toString() + "/dataset", workingDir.toString() + "/dataset",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -151,9 +151,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Dataset", "eu.dnetlib.dhp.schema.oaf.Dataset",
"-outputPath", "-outputPath",
workingDir.toString() + "/dataset", workingDir.toString() + "/dataset",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -258,9 +258,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Dataset", "eu.dnetlib.dhp.schema.oaf.Dataset",
"-outputPath", "-outputPath",
workingDir.toString() + "/dataset", workingDir.toString() + "/dataset",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -349,9 +349,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Publication", "eu.dnetlib.dhp.schema.oaf.Publication",
"-outputPath", "-outputPath",
workingDir.toString() + "/publication", workingDir.toString() + "/publication",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -433,9 +433,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct",
"-outputPath", "-outputPath",
workingDir.toString() + "/orp", workingDir.toString() + "/orp",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -565,9 +565,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Dataset", "eu.dnetlib.dhp.schema.oaf.Dataset",
"-outputPath", "-outputPath",
workingDir.toString() + "/dataset", workingDir.toString() + "/dataset",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -703,9 +703,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Software", "eu.dnetlib.dhp.schema.oaf.Software",
"-outputPath", "-outputPath",
workingDir.toString() + "/software", workingDir.toString() + "/software",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","
@ -814,9 +814,9 @@ public class BulkTagJobTest {
"eu.dnetlib.dhp.schema.oaf.Dataset", "eu.dnetlib.dhp.schema.oaf.Dataset",
"-outputPath", "-outputPath",
workingDir.toString() + "/dataset", workingDir.toString() + "/dataset",
"-isLookupUrl", "-isLookUpUrl",
"http://beta.services.openaire.eu:8280/is/services/isLookUp", "http://beta.services.openaire.eu:8280/is/services/isLookUp",
"-protoMap", "-pathMap",
"{ \"author\" : \"$['author'][*]['fullname']\"," "{ \"author\" : \"$['author'][*]['fullname']\","
+ " \"title\" : \"$['title'][*]['value']\"," + " \"title\" : \"$['title'][*]['value']\","
+ " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\","