lsmyrnaios
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

GetCSV refactoring - changed to use the new class to get and write the csv file

This commit is contained in:
Miriam Baglioni 2021-08-12 18:03:41 +02:00
parent 4317211a2b
commit e9fc3ef3bc
1 changed files with 9 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/ReadCSV.java
View File

@ -1,36 +1,21 @@
package eu.dnetlib.dhp.actionmanager.project.utils; package eu.dnetlib.dhp.actionmanager.project.utils;
import java.io.BufferedWriter; import java.io.*;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.collection.HttpConnector2; import eu.dnetlib.dhp.common.collection.GetCSV;
import eu.dnetlib.dhp.common.collection.HttpConnector2;
/** /**
* Applies the parsing of a csv file and writes the Serialization of it in hdfs * Applies the parsing of a csv file and writes the Serialization of it in hdfs
*/ */
public class ReadCSV implements Closeable { public class ReadCSV {
private static final Log log = LogFactory.getLog(ReadCSV.class);
private final BufferedWriter writer;
private final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final String csvFile;
private final char delimiter;
public static void main(final String[] args) throws Exception { public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser( final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@ -50,57 +35,18 @@ public class ReadCSV implements Closeable {
char del = ';'; char del = ';';
if (delimiter.isPresent()) if (delimiter.isPresent())
del = delimiter.get().charAt(0); del = delimiter.get().charAt(0);
try (final ReadCSV readCSV = new ReadCSV(hdfsPath, hdfsNameNode, fileURL, del)) {
log.info("Getting CSV file...");
readCSV.execute(classForName);
}
}
public void execute(final String classForName)
throws IOException, ClassNotFoundException, IllegalAccessException, InstantiationException {
CSVParser csvParser = new CSVParser();
csvParser
.parse(csvFile, classForName, delimiter)
.stream()
.forEach(this::write);
}
@Override
public void close() throws IOException {
writer.close();
}
public ReadCSV(
final String hdfsPath,
final String hdfsNameNode,
final String fileURL,
char delimiter)
throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode); conf.set("fs.defaultFS", hdfsNameNode);
HttpConnector2 httpConnector = new HttpConnector2();
FileSystem fileSystem = FileSystem.get(conf); FileSystem fileSystem = FileSystem.get(conf);
Path hdfsWritePath = new Path(hdfsPath); BufferedReader reader = new BufferedReader(
new InputStreamReader(new HttpConnector2().getInputSourceAsStream(fileURL)));
if (fileSystem.exists(hdfsWritePath)) { GetCSV.getCsv(fileSystem, reader, hdfsPath, classForName, del);
fileSystem.delete(hdfsWritePath, false);
}
final FSDataOutputStream fos = fileSystem.create(hdfsWritePath);
this.writer = new BufferedWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8)); reader.close();
this.csvFile = httpConnector.getInputSource(fileURL);
this.delimiter = delimiter;
}
protected void write(final Object p) {
try {
writer.write(OBJECT_MAPPER.writeValueAsString(p));
writer.newLine();
} catch (final Exception e) {
throw new RuntimeException(e);
}
} }
} }