diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java index e15fbae65..7cd156982 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java @@ -33,7 +33,6 @@ public class FileCsvCollectorPlugin implements CollectorPlugin { private static final Logger log = LoggerFactory.getLogger(FileCsvCollectorPlugin.class); - // TODO USE HDFS FILESSYSTEM private final FileSystem fileSystem; public FileCsvCollectorPlugin(final FileSystem fileSystem) { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java index 0afa93a45..ba5638e79 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java @@ -43,7 +43,6 @@ public class HttpCsvCollectorPlugin implements CollectorPlugin { private static final Logger log = LoggerFactory.getLogger(HttpCsvCollectorPlugin.class); - // TODO USE HDFS FILESSYSTEM FOR TEMP FILE private final FileSystem fileSystem; private final HttpConnector2 httpConnector; diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java new file mode 100644 index 000000000..36e3a271f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java @@ -0,0 +1,48 @@ +package eu.dnetlib.dhp.collection.plugin.csv; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.dom4j.DocumentException; +import org.dom4j.DocumentHelper; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.dhp.collection.ApiDescriptor; +import eu.dnetlib.dhp.collection.plugin.CollectorPlugin; +import eu.dnetlib.dhp.common.collection.CollectorException; + +@Disabled +class FileCsvCollectorPluginTest { + + private FileCsvCollectorPlugin plugin; + + @Test + void testCollect() throws CollectorException, DocumentException, IOException { + + this.plugin = new FileCsvCollectorPlugin(FileSystem.getLocal(new Configuration())); + + final ApiDescriptor api = new ApiDescriptor(); + api.setBaseUrl("file:///tmp/test.csv"); + api.setProtocol(CollectorPlugin.NAME.fileCSV.name()); + api.getParams().put("header", "true"); + api.getParams().put("identifier", "0"); + api.getParams().put("separator", ","); + api.getParams().put("quote", "\""); + + final List list = this.plugin.collect(api, null).toList(); + assertTrue(list.size() > 0); + + for (final String xml : list) { + System.out.println(xml); + assertTrue(StringUtils.isNotBlank(xml)); + assertTrue(StringUtils.isNotBlank((DocumentHelper.parseText(xml).valueOf("//*[@isId='true']")))); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java new file mode 100644 index 000000000..28156945c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java @@ -0,0 +1,16 @@ +package eu.dnetlib.dhp.collection.plugin.csv; + +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +@Disabled +class HttpCsvCollectorPluginTest { + + @Test + void testCollect() { + fail("Not yet implemented"); + } + +}