przemyslaw.jacewicz
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

changed the save method

This commit is contained in:
Miriam Baglioni 2020-04-16 16:14:15 +02:00
parent 951b13ac46
commit d60fd36046
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dhp-workflows/dhp-propagation/src/main/java/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/SparkResultToOrganizationFromIstRepoJob2.java
View File

@ -35,7 +35,7 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
String jsonConfiguration = IOUtils.toString(SparkCountryPropagationJob2.class String jsonConfiguration = IOUtils.toString(SparkResultToOrganizationFromIstRepoJob2.class
.getResourceAsStream("/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json")); .getResourceAsStream("/eu/dnetlib/dhp/resulttoorganizationfrominstrepo/input_propagationresulaffiliationfrominstrepo_parameters.json"));
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@ -118,9 +118,14 @@ public class SparkResultToOrganizationFromIstRepoJob2 {
.textFile(alreadylinked) .textFile(alreadylinked)
.map(value -> OBJECT_MAPPER.readValue(value, ResultOrganizationSet.class), .map(value -> OBJECT_MAPPER.readValue(value, ResultOrganizationSet.class),
Encoders.bean(ResultOrganizationSet.class)), potentialUpdates) Encoders.bean(ResultOrganizationSet.class)), potentialUpdates)
.toJavaRDD() .toJSON()
.map(r -> OBJECT_MAPPER.writeValueAsString(r)) .write()
.saveAsTextFile(outputPath , GzipCodec.class); .mode(SaveMode.Append)
.option("compression", "gzip")
.text(outputPath);
// .toJavaRDD()
// .map(r -> OBJECT_MAPPER.writeValueAsString(r))
// .saveAsTextFile(outputPath , GzipCodec.class);
} }