D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Incremental support for thanados plugin

This commit is contained in:
Alessia Bardi 2022-05-05 16:13:32 +02:00
parent ae2b4b6388
commit fa71f9a7e1
4 changed files with 39 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dnet-ariadneplus/pom.xml
View File

@ -36,6 +36,11 @@
<artifactId>dnet-msro-service</artifactId>
<version>[7.0.0-SAXONHE-SOLR772-SNAPSHOT, 8.0.0-SAXONHE)</version>
</dependency>
<dependency>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
<version>2.4.8</version>
</dependency>
</dependencies>
<properties>

6
dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosCollectorPlugin.java
View File

@ -4,6 +4,10 @@ import eu.dnetlib.data.collector.plugins.httplist.HttpListIterator;
import eu.dnetlib.rmi.data.CollectorServiceException;
import eu.dnetlib.rmi.data.InterfaceDescriptor;
import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import java.util.Date;
public class ThanadosCollectorPlugin extends AbstractCollectorPlugin {
@ -12,6 +16,6 @@ public class ThanadosCollectorPlugin extends AbstractCollectorPlugin {
final String baseUrl = interfaceDescriptor.getBaseUrl();
final String listAddress = interfaceDescriptor.getParams().get("listUrl");
return () -> new ThanadosIterator(baseUrl, listAddress);
return () -> new ThanadosIterator(baseUrl, listAddress, fromDate);
}
}

26
dnet-ariadneplus/src/main/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIterator.java
View File

@ -5,6 +5,7 @@ import com.google.gson.reflect.TypeToken;
import eu.dnetlib.data.collector.ThreadSafeIterator;
import eu.dnetlib.rmi.data.CollectorServiceRuntimeException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpStatus;
@ -18,6 +19,11 @@ import java.io.IOException;
import java.lang.reflect.Type;
import java.util.List;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;
public class ThanadosIterator extends ThreadSafeIterator {
private static final Log log = LogFactory.getLog(ThanadosIterator.class);
@ -27,12 +33,15 @@ public class ThanadosIterator extends ThreadSafeIterator {
private List<String> identifiers;
private int counter = 0;
private String urlFormat = "%s/%s?format=xml";
private String fromDate;
public ThanadosIterator(final String baseUrl, final String listAddress) {
public ThanadosIterator(final String baseUrl, final String listAddress, final String fromDate) {
try {
this.baseUrl = baseUrl;
this.identifiers = downloadIdentifierList(listAddress);
this.identifiers = downloadIdentifierList(listAddress, fromDate);
this.counter = 0;
this.fromDate = fromDate;
} catch (Exception e) {
throw new CollectorServiceRuntimeException("Error creating iterator", e);
}
@ -59,12 +68,17 @@ public class ThanadosIterator extends ThreadSafeIterator {
}
}
protected List<String> downloadIdentifierList(final String listUrl) {
String list = download(listUrl);
protected List<String> downloadIdentifierList(final String listUrl, final String fromDate) throws ParseException {
String urlToListItems = listUrl;
if(StringUtils.isNotBlank(fromDate))
urlToListItems = listUrl+"/"+fromDate;
log.info("Getting list of items from "+urlToListItems);
String response = download(urlToListItems);
JSONObject map = (JSONObject)(new JSONParser(JSONParser.MODE_PERMISSIVE).parse(response));
final String sites = map.getAsString("\"sites\"");
Gson converter = new Gson();
Type type = new TypeToken<List<String>>(){}.getType();
return converter.fromJson(list, type );
return converter.fromJson(sites, type );
}

11
dnet-ariadneplus/src/test/java/eu/dnetlib/data/collector/plugins/ariadneplus/thanados/ThanadosIteratorTest.java
View File

@ -11,7 +11,7 @@ public class ThanadosIteratorTest {
@Test
public void testDownloadList(){
it = new ThanadosIterator("", sitelist);
it = new ThanadosIterator("", sitelist, "");
it.getIdentifiers().stream().forEach(id -> System.out.println(id));
System.out.println(it.getIdentifiers().size());
}
@ -19,7 +19,7 @@ public class ThanadosIteratorTest {
@Test
public void testDownload(){
int count = 0;
it = new ThanadosIterator("https://thanados.openatlas.eu/api/0.3/subunits", sitelist);
it = new ThanadosIterator("https://thanados.openatlas.eu/api/0.3/subunits", sitelist, "");
while(it.hasNext()){
it.next();
count++;
@ -29,4 +29,11 @@ public class ThanadosIteratorTest {
}
@Test
public void testIncremental(){
it = new ThanadosIterator("", sitelist, "2022-03-12");
it.getIdentifiers().stream().forEach(id -> System.out.println(id));
System.out.println(it.getIdentifiers().size());
}
}