refactoring

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVParser.java

@@ -1,47 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.project.utils;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVRecord;
-import org.apache.commons.lang.reflect.FieldUtils;
-
-/**
- * Reads a generic csv and maps it into classes that mirror its schema
- */
-public class CSVParser {
-
-	public <R> List<R> parse(String csvFile, String classForName)
-		throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException {
-		return parse(csvFile, classForName, ';');
-	}
-
-	public <R> List<R> parse(String csvFile, String classForName, char delimiter)
-		throws ClassNotFoundException, IOException, IllegalAccessException, InstantiationException {
-		final CSVFormat format = CSVFormat.EXCEL
-			.withHeader()
-			.withDelimiter(delimiter)
-			.withQuote('"')
-			.withTrim();
-		List<R> ret = new ArrayList<>();
-		final org.apache.commons.csv.CSVParser parser = org.apache.commons.csv.CSVParser.parse(csvFile, format);
-		final Set<String> headers = parser.getHeaderMap().keySet();
-		Class<?> clazz = Class.forName(classForName);
-		for (CSVRecord csvRecord : parser.getRecords()) {
-
-			@SuppressWarnings("unchecked")
-			final R cc = (R) clazz.newInstance();
-			for (String header : headers) {
-				FieldUtils.writeField(cc, header, csvRecord.get(header), true);
-
-			}
-			ret.add(cc);
-		}
-
-		return ret;
-	}
-}

dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/project/utils/CSVProject.java

@@ -1,200 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.project.utils;
-
-import java.io.Serializable;
-
-/**
- * the mmodel for the projects csv file
- */
-public class CSVProject implements Serializable {
-	private String rcn;
-	private String id;
-	private String acronym;
-	private String status;
-	private String programme;
-	private String topics;
-	private String frameworkProgramme;
-	private String title;
-	private String startDate;
-	private String endDate;
-	private String projectUrl;
-	private String objective;
-	private String totalCost;
-	private String ecMaxContribution;
-	private String call;
-	private String fundingScheme;
-	private String coordinator;
-	private String coordinatorCountry;
-	private String participants;
-	private String participantCountries;
-	private String subjects;
-
-	public String getRcn() {
-		return rcn;
-	}
-
-	public void setRcn(String rcn) {
-		this.rcn = rcn;
-	}
-
-	public String getId() {
-		return id;
-	}
-
-	public void setId(String id) {
-		this.id = id;
-	}
-
-	public String getAcronym() {
-		return acronym;
-	}
-
-	public void setAcronym(String acronym) {
-		this.acronym = acronym;
-	}
-
-	public String getStatus() {
-		return status;
-	}
-
-	public void setStatus(String status) {
-		this.status = status;
-	}
-
-	public String getProgramme() {
-		return programme;
-	}
-
-	public void setProgramme(String programme) {
-		this.programme = programme;
-	}
-
-	public String getTopics() {
-		return topics;
-	}
-
-	public void setTopics(String topics) {
-		this.topics = topics;
-	}
-
-	public String getFrameworkProgramme() {
-		return frameworkProgramme;
-	}
-
-	public void setFrameworkProgramme(String frameworkProgramme) {
-		this.frameworkProgramme = frameworkProgramme;
-	}
-
-	public String getTitle() {
-		return title;
-	}
-
-	public void setTitle(String title) {
-		this.title = title;
-	}
-
-	public String getStartDate() {
-		return startDate;
-	}
-
-	public void setStartDate(String startDate) {
-		this.startDate = startDate;
-	}
-
-	public String getEndDate() {
-		return endDate;
-	}
-
-	public void setEndDate(String endDate) {
-		this.endDate = endDate;
-	}
-
-	public String getProjectUrl() {
-		return projectUrl;
-	}
-
-	public void setProjectUrl(String projectUrl) {
-		this.projectUrl = projectUrl;
-	}
-
-	public String getObjective() {
-		return objective;
-	}
-
-	public void setObjective(String objective) {
-		this.objective = objective;
-	}
-
-	public String getTotalCost() {
-		return totalCost;
-	}
-
-	public void setTotalCost(String totalCost) {
-		this.totalCost = totalCost;
-	}
-
-	public String getEcMaxContribution() {
-		return ecMaxContribution;
-	}
-
-	public void setEcMaxContribution(String ecMaxContribution) {
-		this.ecMaxContribution = ecMaxContribution;
-	}
-
-	public String getCall() {
-		return call;
-	}
-
-	public void setCall(String call) {
-		this.call = call;
-	}
-
-	public String getFundingScheme() {
-		return fundingScheme;
-	}
-
-	public void setFundingScheme(String fundingScheme) {
-		this.fundingScheme = fundingScheme;
-	}
-
-	public String getCoordinator() {
-		return coordinator;
-	}
-
-	public void setCoordinator(String coordinator) {
-		this.coordinator = coordinator;
-	}
-
-	public String getCoordinatorCountry() {
-		return coordinatorCountry;
-	}
-
-	public void setCoordinatorCountry(String coordinatorCountry) {
-		this.coordinatorCountry = coordinatorCountry;
-	}
-
-	public String getParticipants() {
-		return participants;
-	}
-
-	public void setParticipants(String participants) {
-		this.participants = participants;
-	}
-
-	public String getParticipantCountries() {
-		return participantCountries;
-	}
-
-	public void setParticipantCountries(String participantCountries) {
-		this.participantCountries = participantCountries;
-	}
-
-	public String getSubjects() {
-		return subjects;
-	}
-
-	public void setSubjects(String subjects) {
-		this.subjects = subjects;
-	}
-
-}

dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/project/CSVParserTest.java

@@ -1,31 +0,0 @@
-
-package eu.dnetlib.dhp.actionmanager.project;
-
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import eu.dnetlib.dhp.actionmanager.project.utils.CSVParser;
-
-class CSVParserTest {
-
-	@Test
-	void readProgrammeTest() throws Exception {
-
-		String programmecsv = IOUtils
-			.toString(
-				getClass()
-					.getClassLoader()
-					.getResourceAsStream("eu/dnetlib/dhp/actionmanager/project/programme.csv"));
-
-		CSVParser csvParser = new CSVParser();
-
-		List<Object> pl = csvParser.parse(programmecsv, "eu.dnetlib.dhp.actionmanager.project.utils.CSVProgramme");
-
-		Assertions.assertEquals(24, pl.size());
-
-	}
-
-}

dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/hostedbymap/GetCSV.java

@@ -2,27 +2,16 @@
 package eu.dnetlib.dhp.oa.graph.hostedbymap;
 
 import java.io.*;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.opencsv.bean.CsvToBeanBuilder;
 
 import eu.dnetlib.dhp.application.ArgumentApplicationParser;
+import eu.dnetlib.dhp.common.collection.HttpConnector2;
 
 public class GetCSV {
-	private static final Log log = LogFactory.getLog(eu.dnetlib.dhp.oa.graph.hostedbymap.GetCSV.class);
 
 	public static void main(final String[] args) throws Exception {
 		final ArgumentApplicationParser parser = new ArgumentApplicationParser(
@@ -38,66 +27,45 @@ public class GetCSV {
 		final String hdfsPath = parser.get("workingPath");
 		final String hdfsNameNode = parser.get("hdfsNameNode");
 		final String classForName = parser.get("classForName");
+		final String delimiter = Optional
+			.ofNullable(parser.get("delimiter"))
+			.orElse(null);
 		final Boolean shouldReplace = Optional
 			.ofNullable((parser.get("replace")))
 			.map(Boolean::valueOf)
 			.orElse(false);
 
-		URLConnection connection = new URL(fileURL).openConnection();
+		char del = ';';
-		connection
+		if (delimiter != null) {
-			.setRequestProperty(
+			del = delimiter.charAt(0);
-				"User-Agent",
+		}
-				"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
+
-		connection.connect();
+		HttpConnector2 connector2 = new HttpConnector2();
 
 		BufferedReader in = new BufferedReader(
-			new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
+			new InputStreamReader(connector2.getInputSourceAsStream(fileURL)));
 
-		if (shouldReplace) {
+		if (Boolean.TRUE.equals(shouldReplace)) {
-			PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ.csv")));
+			try (PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/replaced.csv")))) {
-			String line = null;
+				String line;
-			while ((line = in.readLine()) != null) {
+				while ((line = in.readLine()) != null) {
-				writer.println(line.replace("\\\"", "\""));
+					writer.println(line.replace("\\\"", "\""));
+				}
 			}
-			writer.close();
+
 			in.close();
-			in = new BufferedReader(new FileReader("/tmp/DOAJ.csv"));
+			in = new BufferedReader(new FileReader("/tmp/replaced.csv"));
 		}
 
 		Configuration conf = new Configuration();
 		conf.set("fs.defaultFS", hdfsNameNode);
 
 		FileSystem fileSystem = FileSystem.get(conf);
-		Path hdfsWritePath = new Path(hdfsPath);
-		FSDataOutputStream fsDataOutputStream = null;
-		if (fileSystem.exists(hdfsWritePath)) {
-			fileSystem.delete(hdfsWritePath, false);
-		}
-		fsDataOutputStream = fileSystem.create(hdfsWritePath);
 
-		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
+		eu.dnetlib.dhp.common.collection.GetCSV.getCsv(fileSystem, in, hdfsPath, classForName, del);
 
-		Class<?> clazz = Class.forName(classForName);
-
-		ObjectMapper mapper = new ObjectMapper();
-
-		new CsvToBeanBuilder(in)
-			.withType(clazz)
-			.withMultilineLimit(1)
-			.build()
-			.parse()
-			.forEach(line -> {
-				try {
-					writer.write(mapper.writeValueAsString(line));
-					writer.newLine();
-				} catch (IOException e) {
-					throw new RuntimeException(e);
-				}
-			});
-
-		writer.close();
 		in.close();
-		if (shouldReplace) {
+		if (Boolean.TRUE.equals(shouldReplace)) {
 			File f = new File("/tmp/DOAJ.csv");
 			f.delete();
 		}

dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/TestReadCSV.java

@@ -1,112 +0,0 @@
-
-package eu.dnetlib.dhp.oa.graph.hostedbymap;
-
-import java.io.*;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.Charset;
-import java.util.List;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.opencsv.bean.CsvToBeanBuilder;
-
-import eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel;
-
-public class TestReadCSV {
-
-	@Test
-	public void testCSVUnibi() throws FileNotFoundException {
-
-		final String sourcePath = getClass()
-			.getResource("/eu/dnetlib/dhp/oa/graph/hostedbymap/unibiGold.csv")
-			.getPath();
-
-		List<UnibiGoldModel> beans = new CsvToBeanBuilder(new FileReader(sourcePath))
-			.withType(UnibiGoldModel.class)
-			.build()
-			.parse();
-
-		Assertions.assertEquals(36, beans.size());
-		Assertions.assertEquals(1, beans.stream().filter(e -> e.getIssn().equals("0001-625X")).count());
-		Assertions
-			.assertTrue(
-				beans
-					.stream()
-					.anyMatch(e -> e.getIssn().equals("0001-625X") && e.getTitle().equals("Acta Mycologica")));
-		Assertions.assertTrue(beans.stream().allMatch(e -> e.getIssn().equals(e.getIssn_l())));
-
-	}
-
-	@Disabled
-	@Test
-	public void testCSVUrlUnibi() throws IOException {
-
-		URL csv = new URL("https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv");
-
-		BufferedReader in = new BufferedReader(new InputStreamReader(csv.openStream()));
-		ObjectMapper mapper = new ObjectMapper();
-
-		new CsvToBeanBuilder(in)
-			.withType(eu.dnetlib.dhp.oa.graph.hostedbymap.model.UnibiGoldModel.class)
-			.build()
-			.parse()
-			.forEach(line ->
-
-			{
-				try {
-					System.out.println(mapper.writeValueAsString(line));
-				} catch (JsonProcessingException e) {
-					e.printStackTrace();
-				}
-			}
-
-			);
-	}
-
-	@Disabled
-	@Test
-	public void testCSVUrlDOAJ() throws IOException {
-
-		URLConnection connection = new URL("https://doaj.org/csv").openConnection();
-		connection
-			.setRequestProperty(
-				"User-Agent",
-				"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
-		connection.connect();
-
-		BufferedReader in = new BufferedReader(
-			new InputStreamReader(connection.getInputStream(), Charset.forName("UTF-8")));
-		// BufferedReader in = new BufferedReader(new FileReader("/tmp/DOAJ.csv"));
-		PrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter("/tmp/DOAJ_1.csv")));
-		String line = null;
-		while ((line = in.readLine()) != null) {
-			writer.println(line.replace("\\\"", "\""));
-		}
-		writer.close();
-		in.close();
-		in = new BufferedReader(new FileReader("/tmp/DOAJ_1.csv"));
-		ObjectMapper mapper = new ObjectMapper();
-
-		new CsvToBeanBuilder(in)
-			.withType(eu.dnetlib.dhp.oa.graph.hostedbymap.model.DOAJModel.class)
-			.withMultilineLimit(1)
-			.build()
-			.parse()
-			.forEach(lline ->
-
-			{
-				try {
-					System.out.println(mapper.writeValueAsString(lline));
-				} catch (JsonProcessingException e) {
-					e.printStackTrace();
-				}
-			}
-
-			);
-	}
-}