From 48c907089325d3bac6d6a566696a974486606890 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 19 Dec 2024 14:48:39 +0100 Subject: [PATCH] tests --- .../plugin/csv/FileCsvCollectorPlugin.java | 1 - .../plugin/csv/HttpCsvCollectorPlugin.java | 1 - .../csv/FileCsvCollectorPluginTest.java | 48 +++++++++++++++++++ .../csv/HttpCsvCollectorPluginTest.java | 16 +++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java index e15fbae65..7cd156982 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java @@ -33,7 +33,6 @@ public class FileCsvCollectorPlugin implements CollectorPlugin { private static final Logger log = LoggerFactory.getLogger(FileCsvCollectorPlugin.class); - // TODO USE HDFS FILESSYSTEM private final FileSystem fileSystem; public FileCsvCollectorPlugin(final FileSystem fileSystem) { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java index 0afa93a45..ba5638e79 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java @@ -43,7 +43,6 @@ public class HttpCsvCollectorPlugin implements CollectorPlugin { private static final Logger log = LoggerFactory.getLogger(HttpCsvCollectorPlugin.class); - // TODO USE HDFS FILESSYSTEM FOR TEMP FILE private final FileSystem fileSystem; private final HttpConnector2 httpConnector; diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java new file mode 100644 index 000000000..36e3a271f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java @@ -0,0 +1,48 @@ +package eu.dnetlib.dhp.collection.plugin.csv; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.dom4j.DocumentException; +import org.dom4j.DocumentHelper; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.dhp.collection.ApiDescriptor; +import eu.dnetlib.dhp.collection.plugin.CollectorPlugin; +import eu.dnetlib.dhp.common.collection.CollectorException; + +@Disabled +class FileCsvCollectorPluginTest { + + private FileCsvCollectorPlugin plugin; + + @Test + void testCollect() throws CollectorException, DocumentException, IOException { + + this.plugin = new FileCsvCollectorPlugin(FileSystem.getLocal(new Configuration())); + + final ApiDescriptor api = new ApiDescriptor(); + api.setBaseUrl("file:///tmp/test.csv"); + api.setProtocol(CollectorPlugin.NAME.fileCSV.name()); + api.getParams().put("header", "true"); + api.getParams().put("identifier", "0"); + api.getParams().put("separator", ","); + api.getParams().put("quote", "\""); + + final List list = this.plugin.collect(api, null).toList(); + assertTrue(list.size() > 0); + + for (final String xml : list) { + System.out.println(xml); + assertTrue(StringUtils.isNotBlank(xml)); + assertTrue(StringUtils.isNotBlank((DocumentHelper.parseText(xml).valueOf("//*[@isId='true']")))); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java new file mode 100644 index 000000000..28156945c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java @@ -0,0 +1,16 @@ +package eu.dnetlib.dhp.collection.plugin.csv; + +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +@Disabled +class HttpCsvCollectorPluginTest { + + @Test + void testCollect() { + fail("Not yet implemented"); + } + +}