D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Remove Relation From AS #466

Merged
claudio.atzori merged 5 commits from webCrawlLessBlackList into beta 2024-07-25 15:50:20 +02:00
Conversation 0 Commits 5 Files Changed 7 +3 -4
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7cff281d3e - Show all commits

7
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/RemoveRelationFromActionSet.java
View File

@ -105,7 +105,7 @@ public class RemoveRelationFromActionSet
Dataset<String> blackList = readBlackList(spark, blackListInputPath) Dataset<String> blackList = readBlackList(spark, blackListInputPath)
.map( .map(
(MapFunction<Row, String>) r -> IdentifierFactory (MapFunction<Row, String>) r -> IdentifierFactory
.idFromPid("50", "doi", ((String) r.getAs("DOI / PMID")).substring(16), true), .idFromPid("50", "doi", ((String) r.getAs("doi")).substring(16), true),
Encoders.STRING()); Encoders.STRING());
// read the old actionset and get the relations in the payload // read the old actionset and get the relations in the payload
@ -151,9 +151,8 @@ public class RemoveRelationFromActionSet
return spark return spark
.read() .read()
.option("header", true) .json(inputPath)
.csv(inputPath) .select("doi");
.select("DOI / PMID");
} }
} }