D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

tests

This commit is contained in:
Michele Artini 2024-12-19 14:48:39 +01:00
parent d8f5be1149
commit 48c9070893
4 changed files with 64 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPlugin.java
View File

@ -33,7 +33,6 @@ public class FileCsvCollectorPlugin implements CollectorPlugin {
private static final Logger log = LoggerFactory.getLogger(FileCsvCollectorPlugin.class);
// TODO USE HDFS FILESSYSTEM
private final FileSystem fileSystem;
public FileCsvCollectorPlugin(final FileSystem fileSystem) {

1
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPlugin.java
View File

@ -43,7 +43,6 @@ public class HttpCsvCollectorPlugin implements CollectorPlugin {
private static final Logger log = LoggerFactory.getLogger(HttpCsvCollectorPlugin.class);
// TODO USE HDFS FILESSYSTEM FOR TEMP FILE
private final FileSystem fileSystem;
private final HttpConnector2 httpConnector;

48
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/FileCsvCollectorPluginTest.java Normal file
View File

@ -0,0 +1,48 @@
package eu.dnetlib.dhp.collection.plugin.csv;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import eu.dnetlib.dhp.collection.ApiDescriptor;
import eu.dnetlib.dhp.collection.plugin.CollectorPlugin;
import eu.dnetlib.dhp.common.collection.CollectorException;
@Disabled
class FileCsvCollectorPluginTest {
private FileCsvCollectorPlugin plugin;
@Test
void testCollect() throws CollectorException, DocumentException, IOException {
this.plugin = new FileCsvCollectorPlugin(FileSystem.getLocal(new Configuration()));
final ApiDescriptor api = new ApiDescriptor();
api.setBaseUrl("file:///tmp/test.csv");
api.setProtocol(CollectorPlugin.NAME.fileCSV.name());
api.getParams().put("header", "true");
api.getParams().put("identifier", "0");
api.getParams().put("separator", ",");
api.getParams().put("quote", "\"");
final List<String> list = this.plugin.collect(api, null).toList();
assertTrue(list.size() > 0);
for (final String xml : list) {
System.out.println(xml);
assertTrue(StringUtils.isNotBlank(xml));
assertTrue(StringUtils.isNotBlank((DocumentHelper.parseText(xml).valueOf("//*[@isId='true']"))));
}
}
}

16
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/csv/HttpCsvCollectorPluginTest.java Normal file
View File

@ -0,0 +1,16 @@
package eu.dnetlib.dhp.collection.plugin.csv;
import static org.junit.jupiter.api.Assertions.fail;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@Disabled
class HttpCsvCollectorPluginTest {
@Test
void testCollect() {
fail("Not yet implemented");
}
}