antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

fixed CSV downloader & tests

This commit is contained in:
Claudio Atzori 2021-08-13 14:17:54 +02:00
parent 17cefe6a97
commit 5f0903d50d
2 changed files with 36 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCSV.java
View File

@ -2,9 +2,12 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap; package eu.dnetlib.dhp.oa.graph.hostedbymap;
import java.io.*; import java.io.*;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import eu.dnetlib.dhp.common.collection.CollectorException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
@ -56,30 +59,37 @@ public class DownloadCSV {
.orElse(DEFAULT_DELIMITER); .orElse(DEFAULT_DELIMITER);
log.info("delimiter {}", delimiter); log.info("delimiter {}", delimiter);
final HttpConnector2 connector2 = new HttpConnector2();
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode); conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf); FileSystem fileSystem = FileSystem.get(conf);
new DownloadCSV().doDownload(fileURL, workingPath, outputFile, classForName, delimiter, fileSystem);
}
protected void doDownload(String fileURL, String workingPath, String outputFile, String classForName, char delimiter, FileSystem fs)
throws IOException, ClassNotFoundException, CollectorException {
final HttpConnector2 connector2 = new HttpConnector2();
final Path path = new Path(workingPath + "/replaced.csv"); final Path path = new Path(workingPath + "/replaced.csv");
try (BufferedReader in = new BufferedReader( try (BufferedReader in = new BufferedReader(
new InputStreamReader(connector2.getInputSourceAsStream(fileURL)))) { new InputStreamReader(connector2.getInputSourceAsStream(fileURL)))) {
try (FSDataOutputStream fos = fileSystem.create(path, true)) { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fs.create(path, true), Charset.defaultCharset()))) {
String line; String line;
while ((line = in.readLine()) != null) { while ((line = in.readLine()) != null) {
fos.writeUTF(line.replace("\\\"", "\"")); writer.write(line.replace("\\\"", "\""));
fos.writeUTF("\n"); writer.newLine();
} }
} }
} }
try (InputStreamReader reader = new InputStreamReader(fileSystem.open(path))) { try (InputStreamReader reader = new InputStreamReader(fs.open(path))) {
GetCSV.getCsv(fileSystem, reader, outputFile, classForName, delimiter); GetCSV.getCsv(fs, reader, outputFile, classForName, delimiter);
} }
} }
} }

42
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCsvTest.java
View File

@ -39,14 +39,16 @@ public class DownloadCsvTest {
String fileURL = "https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv"; String fileURL = "https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv";
GetCSV final String outputFile = workingDir + "/unibi_gold.json";
.getCsv( new DownloadCSV().doDownload(
fs, new BufferedReader( fileURL,
new InputStreamReader(new HttpConnector2().getInputSourceAsStream(fileURL))), workingDir + "/unibi_gold",
workingDir + "/programme", outputFile,
UnibiGoldModel.class.getName()); UnibiGoldModel.class.getName(),
',',
fs);
BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(workingDir + "/programme")))); BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(outputFile))));
String line; String line;
int count = 0; int count = 0;
@ -82,24 +84,16 @@ public class DownloadCsvTest {
String fileURL = "https://doaj.org/csv"; String fileURL = "https://doaj.org/csv";
try (BufferedReader in = new BufferedReader( final String outputFile = workingDir + "/doaj.json";
new InputStreamReader(new HttpConnector2().getInputSourceAsStream(fileURL)))) { new DownloadCSV().doDownload(
try (PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ_1.csv")))) { fileURL,
String line; workingDir + "/doaj",
while ((line = in.readLine()) != null) { outputFile,
writer.println(line.replace("\\\"", "\"")); DOAJModel.class.getName(),
} ',',
} fs);
}
GetCSV BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(outputFile))));
.getCsv(
fs, new BufferedReader(
new FileReader("/tmp/DOAJ_1.csv")),
workingDir + "/programme",
DOAJModel.class.getName());
BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(workingDir + "/programme"))));
String line; String line;
int count = 0; int count = 0;