D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

OAF-store-graph mdstores: save them in text format

This commit is contained in:
Claudio Atzori 2022-01-04 15:02:09 +01:00
parent 3dc48c7ab5
commit 3bd3653be9
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/CollectionUtils.scala
View File

@ -50,14 +50,16 @@ object CollectionUtils {
def saveDataset(dataset: Dataset[Oaf], targetPath: String): Unit = { def saveDataset(dataset: Dataset[Oaf], targetPath: String): Unit = {
implicit val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf]) implicit val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
val mapper = new ObjectMapper
dataset dataset
.flatMap(i => CollectionUtils.fixRelations(i)) .flatMap(i => CollectionUtils.fixRelations(i))
.filter(i => i != null) .filter(i => i != null)
.map(r => mapper.writeValueAsString(r))(Encoders.STRING)
.write .write
.mode(SaveMode.Overwrite) .mode(SaveMode.Overwrite)
.option("compression", "gzip") .option("compression", "gzip")
.json(targetPath) .text(targetPath)
} }
} }