D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all files under Access Protocol > Base URL folder are collected together

This commit is contained in:
Enrico Ottonello 2019-10-21 15:43:29 +02:00
parent fc43b5a85d
commit 4047d32587
2 changed files with 86 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ads/ADSCollectorPlugin.java
View File

@ -1,27 +1,67 @@
package eu.dnetlib.data.collector.plugins.ariadneplus.ads;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.collect.Iterators;
import eu.dnetlib.data.collector.plugins.FileCollectorPlugin;
import eu.dnetlib.data.collector.plugins.filesystem.FileSystemIterator;
import eu.dnetlib.rmi.data.CollectorServiceException;
import eu.dnetlib.rmi.data.InterfaceDescriptor;
public class ADSCollectorPlugin extends FileCollectorPlugin {
private Iterator<String> recordIterator;
private URL basePath;
/** The Constant log. */
private static final Log log = LogFactory.getLog(ADSCollectorPlugin.class);
@Override
public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)
throws CollectorServiceException {
final String baseUrl = interfaceDescriptor.getBaseUrl();
if (baseUrl == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }
String url = "file://".concat(baseUrl);
try {
basePath = new URL(url);
} catch (MalformedURLException mue) {
log.error("Failed collecting from base url " + url, mue);
throw new CollectorServiceException(mue);
}
File baseDir = new File(basePath.getPath());
if (!baseDir.exists()) { throw new CollectorServiceException(String.format("The base ULR %s, does not exist", basePath.getPath())); }
final String suffixToUrl = interfaceDescriptor.getParams().get("suffixToBaseUrl");
log.debug("Start collecting from folder " + baseDir + " ...");
final FileSystemIterator fsi = new FileSystemIterator(baseDir.getAbsolutePath(), "xml");
return () -> {
boolean emptyIterator = true;
while (fsi.hasNext()) {
String nextFilePath = fsi.next();
interfaceDescriptor.setBaseUrl("file://".concat(nextFilePath));
try {
return new ADSIterator(super.collect(interfaceDescriptor, fromDate, untilDate).iterator(), baseUrl, suffixToUrl);
log.debug("Add iterator from " + nextFilePath);
if (emptyIterator) {
recordIterator = new ADSIterator(super.collect(interfaceDescriptor, fromDate, untilDate).iterator(), null, null);
emptyIterator = false;
}
else {
recordIterator = Iterators.concat(recordIterator, new ADSIterator(super.collect(interfaceDescriptor, fromDate, untilDate).iterator(), null, null));
}
} catch (CollectorServiceException e) {
throw new RuntimeException(e);
log.error("Failed collecting from path: " + nextFilePath, e);
}
} ;
}
return new ADSIterable(recordIterator);
}
}

41
dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/ads/ADSIterable.java Normal file
View File

@ -0,0 +1,41 @@
package eu.dnetlib.data.collector.plugins.ariadneplus.ads;
import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* The Class FilesystemIterable.
*
*
*/
public class ADSIterable implements Iterable<String> {
/** The Constant log. */
private static final Log log = LogFactory.getLog(ADSIterable.class);
private Iterator<String> recordIterator;
public ADSIterable(Iterator<String> recordIterator ) {
this.recordIterator = recordIterator;
}
/**
* {@inheritDoc}
*
* @see java.lang.Iterable#iterator()
*/
@Override
public Iterator<String> iterator() {
return recordIterator;
}
public Iterator<String> getRecordIterator() {
return recordIterator;
}
public void setRecordIterator(Iterator<String> recordIterator) {
this.recordIterator = recordIterator;
}
}