forked from D-Net/dnet-hadoop
OAF-store-graph mdstores: save them in text format
This commit is contained in:
parent
174c3037e1
commit
58f8998e3d
|
@ -50,14 +50,16 @@ object CollectionUtils {
|
|||
|
||||
def saveDataset(dataset: Dataset[Oaf], targetPath: String): Unit = {
|
||||
implicit val resultEncoder: Encoder[Oaf] = Encoders.kryo(classOf[Oaf])
|
||||
val mapper = new ObjectMapper
|
||||
|
||||
dataset
|
||||
.flatMap(i => CollectionUtils.fixRelations(i))
|
||||
.filter(i => i != null)
|
||||
.map(r => mapper.writeValueAsString(r))(Encoders.STRING)
|
||||
.write
|
||||
.mode(SaveMode.Overwrite)
|
||||
.option("compression", "gzip")
|
||||
.json(targetPath)
|
||||
.text(targetPath)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue