D-Net
/
dnet-hadoop
13
4
Fork 10
Code Issues 14 Pull Requests 8 Projects 2 Releases 1 Wiki Activity

refactoring

This commit is contained in:
Michele Artini 2024-02-12 12:19:57 +01:00
parent 5add433b74
commit 16766c514e
2 changed files with 37 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/base/BaseCollectorIterator.java
View File

@ -27,7 +27,7 @@ public class BaseCollectorIterator implements Iterator<Element> {
private Object nextElement;
private final BlockingQueue<Object> queue = new LinkedBlockingQueue<>();
private final BlockingQueue<Object> queue = new LinkedBlockingQueue<>(20);
private static final Logger log = LoggerFactory.getLogger(BaseCollectorIterator.class);
@ -78,8 +78,6 @@ public class BaseCollectorIterator implements Iterator<Element> {
log.error("Error processing BASE records", e);
report.put(e.getClass().getName(), e.getMessage());
throw new RuntimeException("Error processing BASE records", e);
} finally {
this.queue.add("__END__"); // I ADD A NOT ELEMENT OBJECT TO INDICATE THE END OF THE QUEUE
}
}
@ -91,8 +89,6 @@ public class BaseCollectorIterator implements Iterator<Element> {
log.error("Error processing BASE records", e);
report.put(e.getClass().getName(), e.getMessage());
throw new RuntimeException("Error processing BASE records", e);
} finally {
this.queue.add("__END__"); // I ADD A NOT ELEMENT OBJECT TO INDICATE THE END OF THE QUEUE
}
}
@ -119,13 +115,16 @@ public class BaseCollectorIterator implements Iterator<Element> {
for (final Object o : doc.selectNodes("//*[local-name()='ListRecords']/*[local-name()='record']")) {
if (o instanceof Element) {
this.queue.add(o);
this.queue.put(((Element) o).detach());
count++;
}
}
}
}
}
this.queue.put("__END__"); // I ADD A NOT ELEMENT OBJECT TO INDICATE THE END OF THE QUEUE
log.info("Total records (written in queue): " + count);
}

33
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/base/BaseCollectorIteratorTest.java
View File

@ -2,6 +2,11 @@ package eu.dnetlib.dhp.collection.plugin.base;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.dom4j.Attribute;
import org.dom4j.Element;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -19,11 +24,37 @@ public class BaseCollectorIteratorTest {
final BaseCollectorIterator iterator = new BaseCollectorIterator("base-sample.tar", new AggregatorReport());
final Map<String, Map<String, String>> collections = new HashMap<>();
while (iterator.hasNext()) {
final Element record = iterator.next();
// System.out.println(record.asXML());
count++;
if ((count % 1000) == 0) {
System.out.println("# Read records: " + count);
}
for (final Object o : record.selectNodes("//*[local-name() = 'collection']")) {
final Element n = (Element) o;
final String collName = n.getText().trim();
if (StringUtils.isNotBlank(collName) && !collections.containsKey(collName)) {
final Map<String, String> collAttrs = new HashMap<>();
for (final Object ao : n.attributes()) {
collAttrs.put(((Attribute) ao).getName(), ((Attribute) ao).getValue());
}
collections.put(collName, collAttrs);
}
}
}
collections.forEach((k, v) -> {
System.out.println(k);
v.forEach((ak, av) -> System.out.println(" - " + ak + "=" + av));
});
assertEquals(30000, count);
}