From 6b0f7cc8b09f93ba6435682137d691fd58bfd75e Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 23 Sep 2024 10:16:53 +0200 Subject: [PATCH 1/2] skip urls with authentication --- .../plugin/osf/OsfPreprintsIterator.java | 5 ++++ .../osf/OsfPreprintsCollectorPluginTest.java | 29 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsIterator.java index 3bb2e2bdf4..9484297d0a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsIterator.java @@ -125,8 +125,13 @@ public class OsfPreprintsIterator implements Iterator { final String xml = JsonUtils.convertToXML(json); return DocumentHelper.parseText(xml); + } catch (final Throwable e) { log.warn(e.getMessage(), e); + if ((e instanceof CollectorException) && e.getMessage().contains("401")) { + final Element root = DocumentHelper.createElement("error_401_authorization_required"); + return DocumentHelper.createDocument(root); + } return downloadUrl(url, attempt + 1); } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java index 708a26acdd..3d8d7a91f9 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java @@ -1,11 +1,15 @@ package eu.dnetlib.dhp.collection.plugin.osf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.HashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; +import org.dom4j.DocumentHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -17,6 +21,7 @@ import eu.dnetlib.dhp.collection.ApiDescriptor; import eu.dnetlib.dhp.common.aggregation.AggregatorReport; import eu.dnetlib.dhp.common.collection.CollectorException; import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.common.collection.HttpConnector2; public class OsfPreprintsCollectorPluginTest { @@ -50,7 +55,7 @@ public class OsfPreprintsCollectorPluginTest { } @Test - // @Disabled + @Disabled void test_limited() throws CollectorException { final AtomicInteger i = new AtomicInteger(0); final Stream stream = this.plugin.collect(this.api, new AggregatorReport()); @@ -83,4 +88,26 @@ public class OsfPreprintsCollectorPluginTest { Assertions.assertTrue(i.get() > 0); } + @Test + // @Disabled + void test_authentication_required() { + final HttpConnector2 connector = new HttpConnector2(); + + try { + final String res = connector.getInputSource("https://api.osf.io/v2/preprints/ydtzx/contributors/?format=json"); + System.out.println(res); + fail(); + } catch (final Throwable e) { + + System.out.println("**** ERROR: " + e.getMessage()); + + if ((e instanceof CollectorException) && e.getMessage().contains("401")) { + System.out.println(" XML: " + DocumentHelper.createDocument().getRootElement().detach()); + } + + assertTrue(e.getMessage().contains("401")); + } + + } + } From 2d7a7a962da50cdc172cdc07ed0d5fb5df78a6f7 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 23 Sep 2024 10:19:36 +0200 Subject: [PATCH 2/2] unit test @Disabled --- .../collection/plugin/osf/OsfPreprintsCollectorPluginTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java index 3d8d7a91f9..efba0c72e5 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/osf/OsfPreprintsCollectorPluginTest.java @@ -89,7 +89,7 @@ public class OsfPreprintsCollectorPluginTest { } @Test - // @Disabled + @Disabled void test_authentication_required() { final HttpConnector2 connector = new HttpConnector2();