diff --git a/dmp-backend/pom.xml b/dmp-backend/pom.xml
index b48f387d6..e946c9595 100644
--- a/dmp-backend/pom.xml
+++ b/dmp-backend/pom.xml
@@ -79,6 +79,12 @@
spring-tx
${org.springframework.version}
+
+ org.springframework
+ spring-context-support
+ ${org.springframework.version}
+
+
@@ -225,7 +231,11 @@
runtime
-
+
+ com.jayway.jsonpath
+ json-path
+ 2.4.0
+
diff --git a/dmp-backend/src/main/java/cache/ResponsesCache.java b/dmp-backend/src/main/java/cache/ResponsesCache.java
new file mode 100644
index 000000000..4b4200a3a
--- /dev/null
+++ b/dmp-backend/src/main/java/cache/ResponsesCache.java
@@ -0,0 +1,43 @@
+package cache;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cache.support.SimpleCacheManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+import org.springframework.cache.guava.GuavaCache;
+
+import com.google.common.cache.CacheBuilder;
+
+
+
+@Component
+@EnableCaching
+public class ResponsesCache {
+
+ public static long HOW_MANY = 30;
+ public static TimeUnit TIME_UNIT = TimeUnit.MINUTES;
+
+
+ @Bean
+ public CacheManager cacheManager() {
+ System.out.print("Loading ResponsesCache...");
+ SimpleCacheManager simpleCacheManager = new SimpleCacheManager();
+ List caches = new ArrayList();
+ caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+ caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+ caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+ caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+ caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+ caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
+ simpleCacheManager.setCaches(caches);
+ System.out.println("OK");
+ return simpleCacheManager;
+ }
+
+}
diff --git a/dmp-backend/src/main/java/helpers/SerializerProvider.java b/dmp-backend/src/main/java/helpers/SerializerProvider.java
index 0d385fa10..c71b1513f 100644
--- a/dmp-backend/src/main/java/helpers/SerializerProvider.java
+++ b/dmp-backend/src/main/java/helpers/SerializerProvider.java
@@ -15,6 +15,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.datatype.hibernate5.Hibernate5Module;
import com.fasterxml.jackson.datatype.hibernate5.Hibernate5Module.Feature;
@@ -40,6 +41,11 @@ public class SerializerProvider {
;
}
+ public static CollectionType constructTypeFor(Class extends Collection> collectionClass, Class> elementClass) {
+ return objectMapper.getTypeFactory().constructCollectionType(collectionClass, elementClass);
+ }
+
+
public static ObjectMapper getJsonSerializer() {
if(objectMapper==null)
initialize();
@@ -67,9 +73,11 @@ public class SerializerProvider {
return getJsonSerializer().readValue(jsonObject, type);
}
- public static T fromJson(String jsonObject, TypeReference type) throws Exception {
- return getJsonSerializer().readValue(jsonObject, type);
+ public static T fromJson(String jsonObject, TypeReference typeRef) throws Exception {
+ return getJsonSerializer().readValue(jsonObject, typeRef);
}
+
+
}
diff --git a/dmp-backend/src/main/java/proxy/config/Loader.java b/dmp-backend/src/main/java/proxy/config/ConfigLoader.java
similarity index 69%
rename from dmp-backend/src/main/java/proxy/config/Loader.java
rename to dmp-backend/src/main/java/proxy/config/ConfigLoader.java
index 4f1b485d7..4a60d8108 100644
--- a/dmp-backend/src/main/java/proxy/config/Loader.java
+++ b/dmp-backend/src/main/java/proxy/config/ConfigLoader.java
@@ -9,26 +9,25 @@ import javax.xml.bind.Unmarshaller;
import com.fasterxml.jackson.databind.ObjectMapper;
-public class Loader {
+public class ConfigLoader {
private ExternalUrls externalUrls;
// public static void main(String [] args) {
-// Loader l = new Loader("file:///home/nikolas/git/OpenAIRE-EUDAT-DMP/dmp-backend/src/main/resources/ExternalUrls.xml");
+// ConfigLoader l = new ConfigLoader("file:///home/nikolas/git/OpenAIRE-EUDAT-DMP/dmp-backend/src/main/resources/ExternalUrls.xml");
// }
- public Loader(String fileUrl) {
- System.out.println("LOADING FILE: "+fileUrl);
+ public ConfigLoader(String fileUrl) {
+ System.out.println("Loaded also config file: "+fileUrl);
InputStream is = null;
try {
JAXBContext jaxbContext = JAXBContext.newInstance(ExternalUrls.class);
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
is = new URL(fileUrl).openStream();
externalUrls = (ExternalUrls) jaxbUnmarshaller.unmarshal(is);
-
- System.out.println(new ObjectMapper().writeValueAsString(externalUrls));
+// System.out.println(new ObjectMapper().writeValueAsString(externalUrls));
}
catch(Exception ex) {
//log the error and shutdown the system (that's a critical error)
@@ -43,6 +42,11 @@ public class Loader {
}
}
}
+
+
+ public ExternalUrls getExternalUrls() {
+ return externalUrls;
+ }
diff --git a/dmp-backend/src/main/java/proxy/config/ExternalUrls.java b/dmp-backend/src/main/java/proxy/config/ExternalUrls.java
index 0d9122e85..302287ff4 100644
--- a/dmp-backend/src/main/java/proxy/config/ExternalUrls.java
+++ b/dmp-backend/src/main/java/proxy/config/ExternalUrls.java
@@ -7,6 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import proxy.config.entities.OrganisationUrls;
+import proxy.config.entities.ProjectUrls;
import proxy.config.entities.RegistryUrls;
import proxy.config.entities.RepositoryUrls;
import proxy.config.entities.ResearcherUrls;
@@ -18,7 +19,7 @@ public class ExternalUrls implements Serializable {
private static final long serialVersionUID = -5076364662014107275L;
-
+ ProjectUrls projects;
RegistryUrls registries;
RepositoryUrls repositories;
ServiceUrls services;
@@ -75,6 +76,17 @@ public class ExternalUrls implements Serializable {
public void setOrganisations(OrganisationUrls organisations) {
this.organisations = organisations;
}
+
+ public ProjectUrls getProjects() {
+ return projects;
+ }
+
+ @XmlElement(name = "projects")
+ public void setProjects(ProjectUrls projects) {
+ this.projects = projects;
+ }
+
+
diff --git a/dmp-backend/src/main/java/proxy/config/UrlConfig.java b/dmp-backend/src/main/java/proxy/config/UrlConfig.java
index deb8d2b06..3619690b6 100644
--- a/dmp-backend/src/main/java/proxy/config/UrlConfig.java
+++ b/dmp-backend/src/main/java/proxy/config/UrlConfig.java
@@ -1,23 +1,25 @@
package proxy.config;
-import java.net.URL;
import javax.xml.bind.annotation.XmlElement;
-
+//private static String DATAPATH = "$['data'][*]['attributes']['pid','name','uri','description']";
+//private static String PAGINATIONPATH = "$['meta']['pagination']['page','pages','count']";
public class UrlConfig {
- private URL url;
private Integer ordinal;
+ private String url;
+ private String dataPath;
+ private String paginationPath;
- public URL getUrl() {
+ public String getUrl() {
return url;
}
@XmlElement(name = "url")
- public void setUrl(URL url) {
+ public void setUrl(String url) {
this.url = url;
}
public Integer getOrdinal() {
@@ -27,6 +29,22 @@ public class UrlConfig {
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
+
+ public String getDataPath() {
+ return dataPath;
+ }
+ @XmlElement(name = "datapath")
+ public void setDataPath(String dataPath) {
+ this.dataPath = dataPath;
+ }
+
+ public String getPaginationPath() {
+ return paginationPath;
+ }
+ @XmlElement(name = "paginationpath")
+ public void setPaginationPath(String paginationPath) {
+ this.paginationPath = paginationPath;
+ }
diff --git a/dmp-backend/src/main/java/proxy/config/entities/ProjectUrls.java b/dmp-backend/src/main/java/proxy/config/entities/ProjectUrls.java
new file mode 100644
index 000000000..3f2b32bd8
--- /dev/null
+++ b/dmp-backend/src/main/java/proxy/config/entities/ProjectUrls.java
@@ -0,0 +1,33 @@
+package proxy.config.entities;
+
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+
+import proxy.config.FetchStrategy;
+import proxy.config.UrlConfig;
+
+
+public class ProjectUrls {
+
+ List urls;
+ FetchStrategy fetchMode;
+
+ public List getUrls() {
+ return urls;
+ }
+ @XmlElementWrapper
+ @XmlElement(name = "urlConfig")
+ public void setUrls(List urls) {
+ this.urls = urls;
+ }
+ public FetchStrategy getFetchMode() {
+ return fetchMode;
+ }
+ @XmlElement(name = "fetchMode")
+ public void setFetchMode(FetchStrategy fetchMode) {
+ this.fetchMode = fetchMode;
+ }
+
+}
\ No newline at end of file
diff --git a/dmp-backend/src/main/java/proxy/config/exceptions/NoURLFound.java b/dmp-backend/src/main/java/proxy/config/exceptions/NoURLFound.java
new file mode 100644
index 000000000..c0c407882
--- /dev/null
+++ b/dmp-backend/src/main/java/proxy/config/exceptions/NoURLFound.java
@@ -0,0 +1,13 @@
+package proxy.config.exceptions;
+
+public class NoURLFound extends Exception {
+
+ private static final long serialVersionUID = -6961447213733280563L;
+
+
+ public NoURLFound(String message) {
+ super(message);
+ }
+
+
+}
diff --git a/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java b/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java
new file mode 100644
index 000000000..a64bfc0ff
--- /dev/null
+++ b/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java
@@ -0,0 +1,196 @@
+package proxy.fetching;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.Cacheable;
+
+import com.google.common.collect.ContiguousSet;
+import com.google.common.collect.DiscreteDomain;
+import com.google.common.collect.Range;
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+
+import proxy.config.ConfigLoader;
+import proxy.config.UrlConfig;
+import proxy.config.exceptions.NoURLFound;
+
+public class RemoteFetcher {
+
+
+ @Autowired private ConfigLoader configLoader;
+
+
+// public static void main(String [] args) throws Exception {
+//
+// String path = "https://eestore.paas2.uninett.no/api/datarepo";
+// String query = "her";
+//
+// RemoteFetcher remoteFetcher = new RemoteFetcher();
+// List