From 7d6aa5de5efd10628066eaacf04c96f0a35cd1ec Mon Sep 17 00:00:00 2001
From: Enrico Ottonello <enrico.ottonello@isti.cnr.it>
Date: Wed, 19 Feb 2020 16:22:22 +0100
Subject: [PATCH] aat json collector that works on a folder

---
 .../ariadneplus/ads/AATCollectorPlugin.java   | 66 +++++++++++++++++++
 ...nContext-ariadneplus-collector-plugins.xml |  2 +-
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ads/AATCollectorPlugin.java

diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ads/AATCollectorPlugin.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ads/AATCollectorPlugin.java
new file mode 100644
index 0000000..5c92765
--- /dev/null
+++ b/dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ads/AATCollectorPlugin.java
@@ -0,0 +1,66 @@
+package eu.dnetlib.data.collector.plugins.ariadneplus.ads;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Iterator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.google.common.collect.Iterators;
+
+import eu.dnetlib.data.collector.plugins.filesystem.FileSystemIterator;
+import eu.dnetlib.rmi.data.CollectorServiceException;
+import eu.dnetlib.rmi.data.InterfaceDescriptor;
+
+
+public class AATCollectorPlugin extends FileJSONCollectorPlugin {
+	
+	private Iterator<String> recordIterator;
+	private URL basePath;
+	
+	/** The Constant log. */
+	private static final Log log = LogFactory.getLog(AATCollectorPlugin.class);
+	
+	@Override
+	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)
+			throws CollectorServiceException {
+
+		final String baseUrl = interfaceDescriptor.getBaseUrl();
+		if (baseUrl == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }
+		String url = "file://".concat(baseUrl);
+		try {
+			basePath = new URL(url);
+		} catch (MalformedURLException mue) {
+			log.error("Failed collecting json aat from base url " + url, mue);
+			throw new CollectorServiceException(mue);
+		}
+
+		File baseDir = new File(basePath.getPath());
+		if (!baseDir.exists()) { throw new CollectorServiceException(String.format("The base ULR %s, does not exist", basePath.getPath())); }
+		
+		log.debug("Start collecting json from folder " + baseDir + " ...");
+		final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), "json");
+		
+		boolean emptyIterator = true;
+		while (fsi.hasNext()) {
+			String nextFilePath = fsi.next();
+			interfaceDescriptor.setBaseUrl("file://".concat(nextFilePath));
+			try {
+				log.debug("Add iterator from " + nextFilePath);
+				if (emptyIterator) {
+					recordIterator = super.collect(interfaceDescriptor, fromDate, untilDate).iterator();
+					emptyIterator = false;
+				}
+				else {
+					recordIterator = Iterators.concat(recordIterator, super.collect(interfaceDescriptor, fromDate, untilDate).iterator());
+				}
+			} catch (CollectorServiceException e) {
+				log.error("Failed collecting json aat from path: " + nextFilePath, e);
+			}
+		}
+		return new ADSIterable(recordIterator);
+	}
+	
+}
diff --git a/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml b/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml
index bc3fe25..cae146f 100644
--- a/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml
+++ b/dnet-ariadneplus/src/main/resources/eu/dnetlib/data/collector/plugins/ariadneplus/applicationContext-ariadneplus-collector-plugins.xml
@@ -17,7 +17,7 @@
 			</bean>
 		</property>
 	</bean>
-	<bean id="aatCollectorPlugin" class="eu.dnetlib.data.collector.plugins.ariadneplus.ads.FileJSONCollectorPlugin">
+	<bean id="aatCollectorPlugin" class="eu.dnetlib.data.collector.plugins.ariadneplus.ads.AATCollectorPlugin">
 		<property name="protocolDescriptor">
 			<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="aat">
 				<property name="params">