D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Moving Download CSV #137

Merged
claudio.atzori merged 28 commits from refactoring_download_csv into beta 2021-08-13 10:41:02 +02:00
Conversation 0 Commits 28 Files Changed 56 +21 -443
6 changed files with 21 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 32fd75691f - Show all commits

47
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java
View File

@ -1,47 +0,0 @@
package eu.dnetlib.dhp.actionmanager.project.utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang.reflect.FieldUtils;
/**
* Reads a generic csv and maps it into classes that mirror its schema
*/
public class CSVParser {
public <R> List<R> parse(String csvFile, String classForName)
throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException {
return parse(csvFile, classForName, ';');
}
public <R> List<R> parse(String csvFile, String classForName, char delimiter)
throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException {
final CSVFormat format = CSVFormat.EXCEL
.withHeader()
.withDelimiter(delimiter)
.withQuote('"')
.withTrim();
List<R> ret = new ArrayList<>();
final org.apache.commons.csv.CSVParser parser = org.apache.commons.csv.CSVParser.parse(csvFile, format);
final Set<String> headers = parser.getHeaderMap().keySet();
Class<?> clazz = Class.forName(classForName);
for (CSVRecord csvRecord : parser.getRecords()) {
@SuppressWarnings("unchecked")
final R cc = (R) clazz.newInstance();
for (String header : headers) {
FieldUtils.writeField(cc, header, csvRecord.get(header), true);
}
ret.add(cc);
}
return ret;
}
}

200
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVProject.java
View File

@ -1,200 +0,0 @@
package eu.dnetlib.dhp.actionmanager.project.utils;
import java.io.Serializable;
/**
* the mmodel for the projects csv file
*/
public class CSVProject implements Serializable {
private String rcn;
private String id;
private String acronym;
private String status;
private String programme;
private String topics;
private String frameworkProgramme;
private String title;
private String startDate;
private String endDate;
private String projectUrl;
private String objective;
private String totalCost;
private String ecMaxContribution;
private String call;
private String fundingScheme;
private String coordinator;
private String coordinatorCountry;
private String participants;
private String participantCountries;
private String subjects;
public String getRcn() {
return rcn;
}
public void setRcn(String rcn) {
this.rcn = rcn;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getAcronym() {
return acronym;
}
public void setAcronym(String acronym) {
this.acronym = acronym;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public String getProgramme() {
return programme;
}
public void setProgramme(String programme) {
this.programme = programme;
}
public String getTopics() {
return topics;
}
public void setTopics(String topics) {
this.topics = topics;
}
public String getFrameworkProgramme() {
return frameworkProgramme;
}
public void setFrameworkProgramme(String frameworkProgramme) {
this.frameworkProgramme = frameworkProgramme;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getStartDate() {
return startDate;
}
public void setStartDate(String startDate) {
this.startDate = startDate;
}
public String getEndDate() {
return endDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
public String getProjectUrl() {
return projectUrl;
}
public void setProjectUrl(String projectUrl) {
this.projectUrl = projectUrl;
}
public String getObjective() {
return objective;
}
public void setObjective(String objective) {
this.objective = objective;
}
public String getTotalCost() {
return totalCost;
}
public void setTotalCost(String totalCost) {
this.totalCost = totalCost;
}
public String getEcMaxContribution() {
return ecMaxContribution;
}
public void setEcMaxContribution(String ecMaxContribution) {
this.ecMaxContribution = ecMaxContribution;
}
public String getCall() {
return call;
}
public void setCall(String call) {
this.call = call;
}
public String getFundingScheme() {
return fundingScheme;
}
public void setFundingScheme(String fundingScheme) {
this.fundingScheme = fundingScheme;
}
public String getCoordinator() {
return coordinator;
}
public void setCoordinator(String coordinator) {
this.coordinator = coordinator;
}
public String getCoordinatorCountry() {
return coordinatorCountry;
}
public void setCoordinatorCountry(String coordinatorCountry) {
this.coordinatorCountry = coordinatorCountry;
}
public String getParticipants() {
return participants;
}
public void setParticipants(String participants) {
this.participants = participants;
}
public String getParticipantCountries() {
return participantCountries;
}
public void setParticipantCountries(String participantCountries) {
this.participantCountries = participantCountries;
}
public String getSubjects() {
return subjects;
}
public void setSubjects(String subjects) {
this.subjects = subjects;
}
}

31
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java
View File

@ -1,31 +0,0 @@
package eu.dnetlib.dhp.actionmanager.project;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import eu.dnetlib.dhp.actionmanager.project.utils.CSVParser;
class CSVParserTest {
@Test
void readProgrammeTest() throws Exception {
String programmecsv = IOUtils
.toString(
getClass()
.getClassLoader()
.getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv"));
CSVParser csvParser = new CSVParser();
List<Object> pl = csvParser.parse(programmecsv, "eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme");
Assertions.assertEquals(24, pl.size());
}
}

BIN
dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/project/preparedProgramme_whole.json.gz
View File

Binary file not shown.

74
dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostedbymap/GetCSV.java
View File

@ -2,27 +2,16 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.opencsv.bean.CsvToBeanBuilder;
import eu.dnetlib.dhp.application.ArgumentApplicationParser;
import eu.dnetlib.dhp.common.collection.HttpConnector2;
public class GetCSV {
private static final Log log = LogFactory.getLog(eu.dnetlib.dhp.oa.graph.hostedbymap.GetCSV.class);
public static void main(final String[] args) throws Exception {
final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@ -38,66 +27,45 @@ public class GetCSV {
final String hdfsPath = parser.get("workingPath");
final String hdfsNameNode = parser.get("hdfsNameNode");
final String classForName = parser.get("classForName");
final String delimiter = Optional
.ofNullable(parser.get("delimiter"))
.orElse(null);
final Boolean shouldReplace = Optional
.ofNullable((parser.get("replace")))
.map(Boolean::valueOf)
.orElse(false);
URLConnection connection = new URL(fileURL).openConnection();
connection
.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
connection.connect();
char del = ';';
if (delimiter != null) {
del = delimiter.charAt(0);
}
HttpConnector2 connector2 = new HttpConnector2();
BufferedReader in = new BufferedReader(
new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
new InputStreamReader(connector2.getInputSourceAsStream(fileURL)));
if (shouldReplace) {
PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ.csv")));
String line = null;
while ((line = in.readLine()) != null) {
writer.println(line.replace("\\\"", "\""));
if (Boolean.TRUE.equals(shouldReplace)) {
try (PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/replaced.csv")))) {
String line;
while ((line = in.readLine()) != null) {
writer.println(line.replace("\\\"", "\""));
}
}
writer.close();
in.close();
in = new BufferedReader(new FileReader("/tmp/DOAJ.csv"));
in = new BufferedReader(new FileReader("/tmp/replaced.csv"));
}
Configuration conf = new Configuration();
conf.set("fs.defaultFS", hdfsNameNode);
FileSystem fileSystem = FileSystem.get(conf);
Path hdfsWritePath = new Path(hdfsPath);
FSDataOutputStream fsDataOutputStream = null;
if (fileSystem.exists(hdfsWritePath)) {
fileSystem.delete(hdfsWritePath, false);
}
fsDataOutputStream = fileSystem.create(hdfsWritePath);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
eu.dnetlib.dhp.common.collection.GetCSV.getCsv(fileSystem, in, hdfsPath, classForName, del);
Class<?> clazz = Class.forName(classForName);
ObjectMapper mapper = new ObjectMapper();
new CsvToBeanBuilder(in)
.withType(clazz)
.withMultilineLimit(1)
.build()
.parse()
.forEach(line -> {
try {
writer.write(mapper.writeValueAsString(line));
writer.newLine();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
writer.close();
in.close();
if (shouldReplace) {
if (Boolean.TRUE.equals(shouldReplace)) {
File f = new File("/tmp/DOAJ.csv");
f.delete();
}

112
dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java
View File

@ -1,112 +0,0 @@
package eu.dnetlib.dhp.oa.graph.hostedbymap;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.opencsv.bean.CsvToBeanBuilder;
import eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel;
public class TestReadCSV {
@Test
public void testCSVUnibi() throws FileNotFoundException {
final String sourcePath = getClass()
.getResource("/eu/dnetlib/dhp/oa/graph/hostedbymap/unibiGold.csv")
.getPath();
List<UnibiGoldModel> beans = new CsvToBeanBuilder(new FileReader(sourcePath))
.withType(UnibiGoldModel.class)
.build()
.parse();
Assertions.assertEquals(36, beans.size());
Assertions.assertEquals(1, beans.stream().filter(e -> e.getIssn().equals("0001-625X")).count());
Assertions
.assertTrue(
beans
.stream()
.anyMatch(e -> e.getIssn().equals("0001-625X") && e.getTitle().equals("Acta Mycologica")));
Assertions.assertTrue(beans.stream().allMatch(e -> e.getIssn().equals(e.getIssn_l())));
}
@Disabled
@Test
public void testCSVUrlUnibi() throws IOException {
URL csv = new URL("https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv");
BufferedReader in = new BufferedReader(new InputStreamReader(csv.openStream()));
ObjectMapper mapper = new ObjectMapper();
new CsvToBeanBuilder(in)
.withType(eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel.class)
.build()
.parse()
.forEach(line ->
{
try {
System.out.println(mapper.writeValueAsString(line));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
);
}
@Disabled
@Test
public void testCSVUrlDOAJ() throws IOException {
URLConnection connection = new URL("https://doaj.org/csv").openConnection();
connection
.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
connection.connect();
BufferedReader in = new BufferedReader(
new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
// BufferedReader in = new BufferedReader(new FileReader("/tmp/DOAJ.csv"));
PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ_1.csv")));
String line = null;
while ((line = in.readLine()) != null) {
writer.println(line.replace("\\\"", "\""));
}
writer.close();
in.close();
in = new BufferedReader(new FileReader("/tmp/DOAJ_1.csv"));
ObjectMapper mapper = new ObjectMapper();
new CsvToBeanBuilder(in)
.withType(eu.dnetlib.dhp.oa.graph.hostedbymap.model.DOAJModel.class)
.withMultilineLimit(1)
.build()
.parse()
.forEach(lline ->
{
try {
System.out.println(mapper.writeValueAsString(lline));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
);
}
}