diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java index dd46ab1f4..3ddaf8ba0 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java @@ -116,7 +116,7 @@ public class HttpConnector2 { String rateLimit = urlConn.getHeaderField(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_LIMIT); String rateRemaining = urlConn.getHeaderField(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING); - if ((rateLimit != null) && (rateRemaining != null) && (Integer.parseInt(rateRemaining) < 2)) { + if ((rateLimit != null) && (rateRemaining != null) && (Integer.parseInt(rateRemaining) < 9)) { if (retryAfter > 0) { backoffAndSleep(retryAfter); } else { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/FileGZipCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/FileGZipCollectorPlugin.java new file mode 100644 index 000000000..8c32e87d7 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/FileGZipCollectorPlugin.java @@ -0,0 +1,23 @@ +public package eu.dnetlib.dhp.collection.plugin; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.net.URL; +import java.util.zip.GZIPInputStream; + +import eu.dnetlib.data.collector.rmi.CollectorServiceException; + +public class FileGZipCollectorPlugin extends AbstractSplittedRecordPlugin { + + @Override + protected BufferedInputStream getBufferedInputStream(final String baseUrl) throws CollectorServiceException { + + try { + GZIPInputStream stream = new GZIPInputStream(new FileInputStream(new URL(baseUrl).getPath())); + return new BufferedInputStream(stream); + } catch (Exception e) { + throw new CollectorServiceException(e); + } + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/original/dris2db.xsl b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/original/dris2db.xsl new file mode 100644 index 000000000..f75e57667 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/transform/scripts/original/dris2db.xsl @@ -0,0 +1,342 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + , + + + + + + + @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + sysimport:crosswalk:entityregistry + + + + + + + + + + + + true + + + + + + + + + + + oai + + + + metadata + + + + + + + //*[local-name()='header']/*[local-name()='identifier'] + + + + + + + + + + format + oai_cerif_openaire + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GB + + + + + + + + + + sysimport:crosswalk:entityregistry + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + \ No newline at end of file