D-Net
/
dhp-graph-dump
4
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

setting the max number of lines per part

This commit is contained in:
Miriam Baglioni 2024-05-06 09:30:17 +02:00
parent 292c69d819
commit ba28d1eb76
6 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/skgif/DumpDatasource.java
View File

@ -99,6 +99,7 @@ public class DumpDatasource implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "datasource"); .json(outputPath + "datasource");
} }

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/skgif/DumpGrant.java
View File

@ -100,6 +100,7 @@ public class DumpGrant implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "grants"); .json(outputPath + "grants");
} }

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/skgif/DumpOrganization.java
View File

@ -122,6 +122,7 @@ public class DumpOrganization implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "organizations"); .json(outputPath + "organizations");
} }

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/skgif/DumpResearchProduct.java
View File

@ -100,6 +100,7 @@ public class DumpResearchProduct implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "products"); .json(outputPath + "products");
} }

1
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/skgif/DumpVenue.java
View File

@ -121,6 +121,7 @@ public class DumpVenue implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "venues"); .json(outputPath + "venues");
} }

2
dump/src/main/java/eu/dnetlib/dhp/oa/graph/dump/skgif/EmitFromEntities.java
View File

@ -167,6 +167,7 @@ public class EmitFromEntities implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "/topics"); .json(outputPath + "/topics");
} }
@ -242,6 +243,7 @@ public class EmitFromEntities implements Serializable {
.write() .write()
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.option("maxRecordsPerFile", 10000)
.json(outputPath + "/persons"); .json(outputPath + "/persons");
} }