D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

fixed a problem in incremental harvesting

This commit is contained in:
Michele Artini 2024-09-17 10:16:28 +02:00
parent 99b7adda0c
commit a2fac78dcc
2 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java
View File

@ -199,12 +199,10 @@ public class Gtr2PublicationsIterator implements Iterator<String> {
}
private DateTime parseDate(final String s) {
// I expect dates in the format 'yyyy-MM-dd'. See class
// eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode
return DateTime.parse(s.substring(0, s.indexOf("T")), simpleDateTimeFormatter);
return DateTime.parse(s.contains("T") ? s.substring(0, s.indexOf("T")) : s, simpleDateTimeFormatter);
}
private boolean isAfter(final String d, final DateTime fromDate) {
return parseDate(d).isAfter(fromDate);
return StringUtils.isNotBlank(d) && parseDate(d).isAfter(fromDate);
}
}

8
dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIteratorTest.java
View File

@ -48,16 +48,16 @@ class Gtr2PublicationsIteratorTest {
public void testOnePage() throws Exception {
final Iterator<String> iterator = new Gtr2PublicationsIterator(baseURL, null, "12", "12", clientParams);
final int count = iterateAndCount(iterator);
assertEquals(21, count);
assertEquals(20, count);
}
@Test
@Disabled
public void testIncrementalHarvestingNoRecords() throws Exception {
System.out.println("incremental Harvesting");
final Iterator<String> iterator = new Gtr2PublicationsIterator(baseURL, "2050-12-12", "11", "13", clientParams);
final Iterator<String> iterator = new Gtr2PublicationsIterator(baseURL, "2050-12-12T", "11", "13", clientParams);
final int count = iterateAndCount(iterator);
assertEquals(1, count);
assertEquals(0, count);
}
@Test
@ -66,7 +66,7 @@ class Gtr2PublicationsIteratorTest {
System.out.println("incremental Harvesting");
final Iterator<String> iterator = new Gtr2PublicationsIterator(baseURL, "2016-11-30", "11", "11", clientParams);
final int count = iterateAndCount(iterator);
assertEquals(21, count);
assertEquals(20, count);
}
@Test