D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

code formatting

This commit is contained in:
Claudio Atzori 2024-05-30 10:21:18 +02:00
parent eadfd8d71d
commit a02f3f0d2b
2 changed files with 52 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java
View File

@ -70,10 +70,11 @@ public class XMLIterator implements Iterator<String> {
super(); super();
this.element = element; this.element = element;
if (element.contains(",")) { if (element.contains(",")) {
elements= Arrays.stream(element.split(",")) elements = Arrays
.filter(StringUtils::isNoneBlank) .stream(element.split(","))
.map(String::toLowerCase) .filter(StringUtils::isNoneBlank)
.collect(Collectors.toList()); .map(String::toLowerCase)
.collect(Collectors.toList());
} }
this.inputStream = inputStream; this.inputStream = inputStream;
this.parser = getParser(); this.parser = getParser();
@ -155,16 +156,16 @@ public class XMLIterator implements Iterator<String> {
XMLEvent peek = parser.peek(); XMLEvent peek = parser.peek();
if (peek != null && peek.isStartElement()) { if (peek != null && peek.isStartElement()) {
String name = peek.asStartElement().getName().getLocalPart(); String name = peek.asStartElement().getName().getLocalPart();
if( isCheckTag(name)) if (isCheckTag(name))
return peek; return peek;
} }
while (parser.hasNext()) { while (parser.hasNext()) {
XMLEvent event= parser.nextEvent(); XMLEvent event = parser.nextEvent();
if (event != null && event.isStartElement()) { if (event != null && event.isStartElement()) {
String name = event.asStartElement().getName().getLocalPart(); String name = event.asStartElement().getName().getLocalPart();
if( isCheckTag(name)) if (isCheckTag(name))
return event; return event;
} }
} }
return null; return null;
@ -181,12 +182,13 @@ public class XMLIterator implements Iterator<String> {
} }
private boolean isCheckTag(final String tagName) { private boolean isCheckTag(final String tagName) {
if (elements!= null) { if (elements != null) {
final String found =elements.stream() final String found = elements
.filter(e -> e.equalsIgnoreCase(tagName)) .stream()
.findFirst() .filter(e -> e.equalsIgnoreCase(tagName))
.orElse(null); .findFirst()
if (found!= null) .orElse(null);
if (found != null)
return true; return true;
} else { } else {
if (element.equalsIgnoreCase(tagName)) { if (element.equalsIgnoreCase(tagName)) {

69
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java
View File

@ -1,9 +1,11 @@
package eu.dnetlib.dhp.collection.plugin.file; package eu.dnetlib.dhp.collection.plugin.file;
import java.io.IOException;
import java.util.HashMap;
import java.util.Objects;
import java.util.stream.Stream;
import eu.dnetlib.dhp.collection.ApiDescriptor;
import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
import eu.dnetlib.dhp.common.collection.CollectorException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.*; import org.junit.jupiter.api.*;
@ -12,52 +14,51 @@ import org.mockito.junit.jupiter.MockitoExtension;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; import eu.dnetlib.dhp.collection.ApiDescriptor;
import java.util.HashMap; import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
import java.util.Objects; import eu.dnetlib.dhp.common.collection.CollectorException;
import java.util.stream.Stream;
@TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public class FileGZipMultipleNodeTest { public class FileGZipMultipleNodeTest {
private static final Logger log = LoggerFactory.getLogger(FileGZipCollectorPluginTest.class); private static final Logger log = LoggerFactory.getLogger(FileGZipCollectorPluginTest.class);
private final ApiDescriptor api = new ApiDescriptor(); private final ApiDescriptor api = new ApiDescriptor();
private FileGZipCollectorPlugin plugin; private FileGZipCollectorPlugin plugin;
private static final String SPLIT_ON_ELEMENT = "incollection,article"; private static final String SPLIT_ON_ELEMENT = "incollection,article";
@BeforeEach @BeforeEach
public void setUp() throws IOException { public void setUp() throws IOException {
final String gzipFile = Objects final String gzipFile = Objects
.requireNonNull( .requireNonNull(
this this
.getClass() .getClass()
.getResource("/eu/dnetlib/dhp/collection/plugin/file/dblp.gz")) .getResource("/eu/dnetlib/dhp/collection/plugin/file/dblp.gz"))
.getFile(); .getFile();
api.setBaseUrl(gzipFile); api.setBaseUrl(gzipFile);
HashMap<String, String> params = new HashMap<>(); HashMap<String, String> params = new HashMap<>();
params.put("splitOnElement", SPLIT_ON_ELEMENT); params.put("splitOnElement", SPLIT_ON_ELEMENT);
api.setParams(params); api.setParams(params);
FileSystem fs = FileSystem.get(new Configuration()); FileSystem fs = FileSystem.get(new Configuration());
plugin = new FileGZipCollectorPlugin(fs); plugin = new FileGZipCollectorPlugin(fs);
} }
@Test @Test
void test() throws CollectorException { void test() throws CollectorException {
final Stream<String> stream = plugin.collect(api, new AggregatorReport()); final Stream<String> stream = plugin.collect(api, new AggregatorReport());
stream.limit(10).forEach(s -> { stream.limit(10).forEach(s -> {
Assertions.assertTrue(s.length() > 0); Assertions.assertTrue(s.length() > 0);
log.info(s); log.info(s);
}); });
} }
} }