D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

master #11

Manually merged
claudio.atzori merged 275 commits from :master into enrichment_wfs 2020-05-11 15:14:56 +02:00
Conversation 1 Commits 275 Files Changed 156 +9 -4
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit d60fd36046 - Show all commits

13
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java
View File

@ -35,7 +35,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils.toString(SparkCountryPropagationJob2.class String jsonConfiguration = IOUtils.toString(SparkResultToOrganizationFromIstRepoJob2.class
.getResourceAsStream("/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json")); .getResourceAsStream("/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@ -118,9 +118,14 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
.textFile(alreadylinked) .textFile(alreadylinked)
.map(value -> OBJECT_MAPPER.readValue(value, ResultOrganizationSet.class), .map(value -> OBJECT_MAPPER.readValue(value, ResultOrganizationSet.class),
Encoders.bean(ResultOrganizationSet.class)), potentialUpdates) Encoders.bean(ResultOrganizationSet.class)), potentialUpdates)
.toJavaRDD() .toJSON()
.map(r -> OBJECT_MAPPER.writeValueAsString(r)) .write()
.saveAsTextFile(outputPath , GzipCodec.class); .mode(SaveMode.Append)
.option("compression", "gzip")
.text(outputPath);
// .toJavaRDD()
// .map(r -> OBJECT_MAPPER.writeValueAsString(r))
// .saveAsTextFile(outputPath , GzipCodec.class);
} }