D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

Moving Download CSV #137

Merged
claudio.atzori merged 28 commits from refactoring_download_csv into beta 2021-08-13 10:41:02 +02:00
Conversation 0 Commits 28 Files Changed 56 +46 -46
1 changed files with 46 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f3d575f749 - Show all commits

92
dhp-common/src/test/java/eu/dnetlib/dhp/common/collection/GetCSVTest.java
View File

@ -26,15 +26,6 @@ public class GetCSVTest {
private static LocalFileSystem fs; private static LocalFileSystem fs;
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files
.createTempDirectory(GetCSVTest.class.getSimpleName())
.toString();
fs = FileSystem.getLocal(new Configuration());
}
@Disabled @Disabled
@Test @Test
void getProgrammeFileTest() throws Exception { void getProgrammeFileTest() throws Exception {
@ -42,11 +33,11 @@ public class GetCSVTest {
String fileURL = "https://cordis.europa.eu/data/reference/cordisref-h2020programmes.csv"; String fileURL = "https://cordis.europa.eu/data/reference/cordisref-h2020programmes.csv";
GetCSV GetCSV
.getCsv( .getCsv(
fs, new BufferedReader( fs, new BufferedReader(
new InputStreamReader(new HttpConnector2().getInputSourceAsStream(fileURL))), new InputStreamReader(new HttpConnector2().getInputSourceAsStream(fileURL))),
workingDir + "/programme", workingDir + "/programme",
"eu.dnetlib.dhp.common.collection.models.CSVProgramme", ';'); "eu.dnetlib.dhp.common.collection.models.CSVProgramme", ';');
BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(workingDir + "/programme")))); BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(workingDir + "/programme"))));
@ -57,39 +48,39 @@ public class GetCSVTest {
if (count == 0) { if (count == 0) {
Assertions.assertTrue(csvp.getCode().equals("H2020-EU.5.f.")); Assertions.assertTrue(csvp.getCode().equals("H2020-EU.5.f."));
Assertions Assertions
.assertTrue( .assertTrue(
csvp csvp
.getTitle() .getTitle()
.startsWith( .startsWith(
"Develop the governance for the advancement of responsible research and innovation by all stakeholders")); "Develop the governance for the advancement of responsible research and innovation by all stakeholders"));
Assertions Assertions
.assertTrue(csvp.getTitle().endsWith("promote an ethics framework for research and innovation")); .assertTrue(csvp.getTitle().endsWith("promote an ethics framework for research and innovation"));
Assertions.assertTrue(csvp.getShortTitle().equals("")); Assertions.assertTrue(csvp.getShortTitle().equals(""));
Assertions.assertTrue(csvp.getLanguage().equals("en")); Assertions.assertTrue(csvp.getLanguage().equals("en"));
} }
if (count == 28) { if (count == 28) {
Assertions.assertTrue(csvp.getCode().equals("H2020-EU.3.5.4.")); Assertions.assertTrue(csvp.getCode().equals("H2020-EU.3.5.4."));
Assertions Assertions
.assertTrue( .assertTrue(
csvp csvp
.getTitle() .getTitle()
.equals( .equals(
"Grundlagen für den Übergang zu einer umweltfreundlichen Wirtschaft und Gesellschaft durch Öko-Innovation")); "Grundlagen für den Übergang zu einer umweltfreundlichen Wirtschaft und Gesellschaft durch Öko-Innovation"));
Assertions Assertions
.assertTrue(csvp.getShortTitle().equals("A green economy and society through eco-innovation")); .assertTrue(csvp.getShortTitle().equals("A green economy and society through eco-innovation"));
Assertions.assertTrue(csvp.getLanguage().equals("de")); Assertions.assertTrue(csvp.getLanguage().equals("de"));
} }
if (count == 229) { if (count == 229) {
Assertions.assertTrue(csvp.getCode().equals("H2020-EU.3.2.")); Assertions.assertTrue(csvp.getCode().equals("H2020-EU.3.2."));
Assertions Assertions
.assertTrue( .assertTrue(
csvp csvp
.getTitle() .getTitle()
.equals( .equals(
"SOCIETAL CHALLENGES - Food security, sustainable agriculture and forestry, marine, maritime and inland water research, and the bioeconomy")); "SOCIETAL CHALLENGES - Food security, sustainable agriculture and forestry, marine, maritime and inland water research, and the bioeconomy"));
Assertions Assertions
.assertTrue( .assertTrue(
csvp.getShortTitle().equals("Food, agriculture, forestry, marine research and bioeconomy")); csvp.getShortTitle().equals("Food, agriculture, forestry, marine research and bioeconomy"));
Assertions.assertTrue(csvp.getLanguage().equals("en")); Assertions.assertTrue(csvp.getLanguage().equals("en"));
} }
Assertions.assertTrue(csvp.getCode() != null); Assertions.assertTrue(csvp.getCode() != null);
@ -100,6 +91,15 @@ public class GetCSVTest {
Assertions.assertEquals(767, count); Assertions.assertEquals(767, count);
} }
@BeforeAll
public static void beforeAll() throws IOException {
workingDir = Files
.createTempDirectory(GetCSVTest.class.getSimpleName())
.toString();
fs = FileSystem.getLocal(new Configuration());
}
@Disabled @Disabled
@Test @Test
void getProjectFileTest() throws IOException, CollectorException, ClassNotFoundException { void getProjectFileTest() throws IOException, CollectorException, ClassNotFoundException {
@ -217,29 +217,29 @@ public class GetCSVTest {
while ((line = in.readLine()) != null) { while ((line = in.readLine()) != null) {
DOAJModel doaj = new ObjectMapper().readValue(line, DOAJModel.class); DOAJModel doaj = new ObjectMapper().readValue(line, DOAJModel.class);
if (count == 0) { if (count == 0) {
Assertions.assertTrue(doaj.getIssn().equals("0001-3765")); Assertions.assertEquals("0001-3765", doaj.getIssn());
Assertions.assertTrue(doaj.getEissn().equals("1678-2690")); Assertions.assertEquals("1678-2690", doaj.getEissn());
Assertions.assertTrue(doaj.getJournalTitle().equals("Anais da Academia Brasileira de Ciências")); Assertions.assertEquals("Anais da Academia Brasileira de Ciências", doaj.getJournalTitle());
} }
if (count == 7902) { if (count == 7904) {
System.out.println(new ObjectMapper().writeValueAsString(doaj));
Assertions.assertTrue(doaj.getIssn().equals("")); Assertions.assertEquals("",doaj.getIssn());
Assertions.assertTrue(doaj.getEissn().equals("2055-7159")); Assertions.assertEquals("2055-7159", doaj.getEissn());
Assertions.assertTrue(doaj.getJournalTitle().equals("BJR|case reports")); Assertions.assertEquals("BJR|case reports", doaj.getJournalTitle());
} }
if (count == 16703) { if (count == 16707) {
Assertions.assertTrue(doaj.getIssn().equals("")); Assertions.assertEquals("",doaj.getIssn());
Assertions.assertTrue(doaj.getEissn().equals("2788-6298")); Assertions.assertEquals("2788-6298",doaj.getEissn());
Assertions Assertions
.assertTrue(doaj.getJournalTitle().equals("Teacher Education through Flexible Learning in Africa")); .assertEquals("Teacher Education through Flexible Learning in Africa", doaj.getJournalTitle());
} }
count += 1; count += 1;
} }
Assertions.assertEquals(16709, count); Assertions.assertEquals(16713, count);
} }
} }