forked from D-Net/dnet-hadoop
36 lines
980 B
Java
36 lines
980 B
Java
|
|
package eu.dnetlib.dhp.common;
|
|
|
|
import org.apache.spark.sql.DataFrameWriter;
|
|
import org.apache.spark.sql.Dataset;
|
|
import org.apache.spark.sql.SaveMode;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
|
import eu.dnetlib.dhp.schema.oaf.Oaf;
|
|
|
|
public class GraphSupport {
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(GraphSupport.class);
|
|
|
|
private static <T extends Oaf> void saveGraphTable(Dataset<T> dataset, Class<T> clazz, String outputGraph,
|
|
eu.dnetlib.dhp.common.SaveMode saveMode) {
|
|
|
|
log.info("saving graph in {} mode to {}", outputGraph, saveMode.toString());
|
|
|
|
final DataFrameWriter<T> writer = dataset.write().mode(SaveMode.Overwrite);
|
|
switch (saveMode) {
|
|
case JSON:
|
|
writer.option("compression", "gzip").json(outputGraph);
|
|
break;
|
|
case PARQUET:
|
|
final String db_table = ModelSupport.tableIdentifier(outputGraph, clazz);
|
|
writer.saveAsTable(db_table);
|
|
break;
|
|
}
|
|
|
|
}
|
|
|
|
}
|