98 lines
3.1 KiB
Java
98 lines
3.1 KiB
Java
|
|
package eu.dnetlib.dhp.oa.graph.dump.subset;
|
|
|
|
import java.io.BufferedWriter;
|
|
import java.io.Closeable;
|
|
import java.io.IOException;
|
|
import java.io.OutputStreamWriter;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.sql.ResultSet;
|
|
import java.sql.SQLException;
|
|
import java.util.function.Function;
|
|
|
|
import org.apache.commons.io.IOUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.commons.logging.Log;
|
|
import org.apache.commons.logging.LogFactory;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
|
import org.apache.hadoop.fs.FileSystem;
|
|
import org.apache.hadoop.fs.Path;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.mongodb.DBCursor;
|
|
|
|
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
|
|
import eu.dnetlib.dhp.common.DbClient;
|
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
|
|
|
public class ReadMasterDuplicateFromDB {
|
|
|
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
|
|
|
private static final String QUERY = "SELECT id as master, duplicate FROM dsm_dedup_services; ";
|
|
|
|
public static void main(final String[] args) throws Exception {
|
|
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
|
|
IOUtils
|
|
.toString(
|
|
ReadMasterDuplicateFromDB.class
|
|
.getResourceAsStream(
|
|
"/eu/dnetlib/dhp/oa/graph/dump/datasourcemaster_parameters.json")));
|
|
|
|
parser.parseArgument(args);
|
|
|
|
final String dbUrl = parser.get("postgresUrl");
|
|
final String dbUser = parser.get("postgresUser");
|
|
final String dbPassword = parser.get("postgresPassword");
|
|
final String hdfsPath = parser.get("hdfsPath");
|
|
final String hdfsNameNode = parser.get("hdfsNameNode");
|
|
|
|
Configuration conf = new Configuration();
|
|
conf.set("fs.defaultFS", hdfsNameNode);
|
|
|
|
FileSystem fileSystem = FileSystem.get(conf);
|
|
Path hdfsWritePath = new Path(hdfsPath);
|
|
FSDataOutputStream fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
|
|
|
execute(dbUrl, dbUser, dbPassword, fsDataOutputStream);
|
|
|
|
}
|
|
|
|
private static void execute(String dbUrl, String dbUser, String dbPassword, FSDataOutputStream fos) {
|
|
try (DbClient dbClient = new DbClient(dbUrl, dbUser, dbPassword)) {
|
|
try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8))) {
|
|
dbClient.processResults(QUERY, rs -> writeMap(datasourceMasterMap(rs), writer));
|
|
}
|
|
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public static MasterDuplicate datasourceMasterMap(ResultSet rs) {
|
|
try {
|
|
MasterDuplicate dm = new MasterDuplicate();
|
|
String duplicate = rs.getString("duplicate");
|
|
dm.setDuplicate(OafMapperUtils.createOpenaireId(10, duplicate, true));
|
|
String master = rs.getString("master");
|
|
dm.setMaster(OafMapperUtils.createOpenaireId(10, master, true));
|
|
|
|
return dm;
|
|
|
|
} catch (final SQLException e) {
|
|
throw new RuntimeException(e);
|
|
}
|
|
}
|
|
|
|
protected static void writeMap(final MasterDuplicate dm, BufferedWriter writer) {
|
|
try {
|
|
writer.write(OBJECT_MAPPER.writeValueAsString(dm));
|
|
writer.newLine();
|
|
} catch (final IOException e) {
|
|
throw new RuntimeException(e);
|
|
}
|
|
}
|
|
|
|
}
|