dimitris.pierrakos
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

[Enrichment Step] get rid of hive

This commit is contained in:
Miriam Baglioni 2022-04-14 08:50:37 +02:00
parent 7501e823ed
commit 30e0f60ac8
2 changed files with 2 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/orcidtoresultfromsemrel/PrepareResultOrcidAssociationStep1.java
View File

@ -80,7 +80,7 @@ public class PrepareResultOrcidAssociationStep1 {
conf,
isSparkSessionManaged,
spark -> {
removeOutputDir(spark, outputPath);
// removeOutputDir(spark, outputPath);
prepareInfo(
spark, inputPath, outputPath, resultType, resultClazz, allowedsemrel, allowedPids);
});
@ -96,14 +96,6 @@ public class PrepareResultOrcidAssociationStep1 {
List<String> allowedPids) {
final String inputResultPath = inputPath + "/" + resultType;
readPath(spark, inputPath + "/relation", Relation.class)
.filter(
(FilterFunction<Relation>) r -> !r.getDataInfo().getDeletedbyinference()
&& allowedsemrel.contains(r.getRelClass().toLowerCase()))
.write()
.mode(SaveMode.Overwrite)
.option("compression", "gzip")
.json(outputPath + "/" + resultType + "/relationSubset");
Dataset<Relation> relation = readPath(spark, outputPath + "/relationSubset", Relation.class);

2
dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/wf/subworkflows/orcidtoresultfromsemrel/oozie_app/workflow.xml
View File

@ -14,7 +14,7 @@
</property>
</parameters>
<start to="fork_prepare_assoc_step1"/>
<start to="prepare_relations"/>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>