From a2fac78dcc77ca3647765f73c4cbe86f40614dc6 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 17 Sep 2024 10:16:28 +0200 Subject: [PATCH] fixed a problem in incremental harvesting --- .../collection/plugin/gtr2/Gtr2PublicationsIterator.java | 6 ++---- .../plugin/gtr2/Gtr2PublicationsIteratorTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java index c3c4cbac0d..9b122bbe69 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIterator.java @@ -199,12 +199,10 @@ public class Gtr2PublicationsIterator implements Iterator { } private DateTime parseDate(final String s) { - // I expect dates in the format 'yyyy-MM-dd'. See class - // eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode - return DateTime.parse(s.substring(0, s.indexOf("T")), simpleDateTimeFormatter); + return DateTime.parse(s.contains("T") ? s.substring(0, s.indexOf("T")) : s, simpleDateTimeFormatter); } private boolean isAfter(final String d, final DateTime fromDate) { - return parseDate(d).isAfter(fromDate); + return StringUtils.isNotBlank(d) && parseDate(d).isAfter(fromDate); } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIteratorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIteratorTest.java index 737ef0dcac..f02feb0eff 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIteratorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/gtr2/Gtr2PublicationsIteratorTest.java @@ -48,16 +48,16 @@ class Gtr2PublicationsIteratorTest { public void testOnePage() throws Exception { final Iterator iterator = new Gtr2PublicationsIterator(baseURL, null, "12", "12", clientParams); final int count = iterateAndCount(iterator); - assertEquals(21, count); + assertEquals(20, count); } @Test @Disabled public void testIncrementalHarvestingNoRecords() throws Exception { System.out.println("incremental Harvesting"); - final Iterator iterator = new Gtr2PublicationsIterator(baseURL, "2050-12-12", "11", "13", clientParams); + final Iterator iterator = new Gtr2PublicationsIterator(baseURL, "2050-12-12T", "11", "13", clientParams); final int count = iterateAndCount(iterator); - assertEquals(1, count); + assertEquals(0, count); } @Test @@ -66,7 +66,7 @@ class Gtr2PublicationsIteratorTest { System.out.println("incremental Harvesting"); final Iterator iterator = new Gtr2PublicationsIterator(baseURL, "2016-11-30", "11", "11", clientParams); final int count = iterateAndCount(iterator); - assertEquals(21, count); + assertEquals(20, count); } @Test