diff --git a/dmp-backend/pom.xml b/dmp-backend/pom.xml index b48f387d6..e946c9595 100644 --- a/dmp-backend/pom.xml +++ b/dmp-backend/pom.xml @@ -79,6 +79,12 @@ spring-tx ${org.springframework.version} + + org.springframework + spring-context-support + ${org.springframework.version} + + @@ -225,7 +231,11 @@ runtime - + + com.jayway.jsonpath + json-path + 2.4.0 + diff --git a/dmp-backend/src/main/java/cache/ResponsesCache.java b/dmp-backend/src/main/java/cache/ResponsesCache.java new file mode 100644 index 000000000..4b4200a3a --- /dev/null +++ b/dmp-backend/src/main/java/cache/ResponsesCache.java @@ -0,0 +1,43 @@ +package cache; + + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.support.SimpleCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.springframework.cache.guava.GuavaCache; + +import com.google.common.cache.CacheBuilder; + + + +@Component +@EnableCaching +public class ResponsesCache { + + public static long HOW_MANY = 30; + public static TimeUnit TIME_UNIT = TimeUnit.MINUTES; + + + @Bean + public CacheManager cacheManager() { + System.out.print("Loading ResponsesCache..."); + SimpleCacheManager simpleCacheManager = new SimpleCacheManager(); + List caches = new ArrayList(); + caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + simpleCacheManager.setCaches(caches); + System.out.println("OK"); + return simpleCacheManager; + } + +} diff --git a/dmp-backend/src/main/java/helpers/SerializerProvider.java b/dmp-backend/src/main/java/helpers/SerializerProvider.java index 0d385fa10..c71b1513f 100644 --- a/dmp-backend/src/main/java/helpers/SerializerProvider.java +++ b/dmp-backend/src/main/java/helpers/SerializerProvider.java @@ -15,6 +15,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.datatype.hibernate5.Hibernate5Module; import com.fasterxml.jackson.datatype.hibernate5.Hibernate5Module.Feature; @@ -40,6 +41,11 @@ public class SerializerProvider { ; } + public static CollectionType constructTypeFor(Class collectionClass, Class elementClass) { + return objectMapper.getTypeFactory().constructCollectionType(collectionClass, elementClass); + } + + public static ObjectMapper getJsonSerializer() { if(objectMapper==null) initialize(); @@ -67,9 +73,11 @@ public class SerializerProvider { return getJsonSerializer().readValue(jsonObject, type); } - public static T fromJson(String jsonObject, TypeReference type) throws Exception { - return getJsonSerializer().readValue(jsonObject, type); + public static T fromJson(String jsonObject, TypeReference typeRef) throws Exception { + return getJsonSerializer().readValue(jsonObject, typeRef); } + + } diff --git a/dmp-backend/src/main/java/proxy/config/Loader.java b/dmp-backend/src/main/java/proxy/config/ConfigLoader.java similarity index 69% rename from dmp-backend/src/main/java/proxy/config/Loader.java rename to dmp-backend/src/main/java/proxy/config/ConfigLoader.java index 4f1b485d7..4a60d8108 100644 --- a/dmp-backend/src/main/java/proxy/config/Loader.java +++ b/dmp-backend/src/main/java/proxy/config/ConfigLoader.java @@ -9,26 +9,25 @@ import javax.xml.bind.Unmarshaller; import com.fasterxml.jackson.databind.ObjectMapper; -public class Loader { +public class ConfigLoader { private ExternalUrls externalUrls; // public static void main(String [] args) { -// Loader l = new Loader("file:///home/nikolas/git/OpenAIRE-EUDAT-DMP/dmp-backend/src/main/resources/ExternalUrls.xml"); +// ConfigLoader l = new ConfigLoader("file:///home/nikolas/git/OpenAIRE-EUDAT-DMP/dmp-backend/src/main/resources/ExternalUrls.xml"); // } - public Loader(String fileUrl) { - System.out.println("LOADING FILE: "+fileUrl); + public ConfigLoader(String fileUrl) { + System.out.println("Loaded also config file: "+fileUrl); InputStream is = null; try { JAXBContext jaxbContext = JAXBContext.newInstance(ExternalUrls.class); Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); is = new URL(fileUrl).openStream(); externalUrls = (ExternalUrls) jaxbUnmarshaller.unmarshal(is); - - System.out.println(new ObjectMapper().writeValueAsString(externalUrls)); +// System.out.println(new ObjectMapper().writeValueAsString(externalUrls)); } catch(Exception ex) { //log the error and shutdown the system (that's a critical error) @@ -43,6 +42,11 @@ public class Loader { } } } + + + public ExternalUrls getExternalUrls() { + return externalUrls; + } diff --git a/dmp-backend/src/main/java/proxy/config/ExternalUrls.java b/dmp-backend/src/main/java/proxy/config/ExternalUrls.java index 0d9122e85..302287ff4 100644 --- a/dmp-backend/src/main/java/proxy/config/ExternalUrls.java +++ b/dmp-backend/src/main/java/proxy/config/ExternalUrls.java @@ -7,6 +7,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import proxy.config.entities.OrganisationUrls; +import proxy.config.entities.ProjectUrls; import proxy.config.entities.RegistryUrls; import proxy.config.entities.RepositoryUrls; import proxy.config.entities.ResearcherUrls; @@ -18,7 +19,7 @@ public class ExternalUrls implements Serializable { private static final long serialVersionUID = -5076364662014107275L; - + ProjectUrls projects; RegistryUrls registries; RepositoryUrls repositories; ServiceUrls services; @@ -75,6 +76,17 @@ public class ExternalUrls implements Serializable { public void setOrganisations(OrganisationUrls organisations) { this.organisations = organisations; } + + public ProjectUrls getProjects() { + return projects; + } + + @XmlElement(name = "projects") + public void setProjects(ProjectUrls projects) { + this.projects = projects; + } + + diff --git a/dmp-backend/src/main/java/proxy/config/UrlConfig.java b/dmp-backend/src/main/java/proxy/config/UrlConfig.java index deb8d2b06..3619690b6 100644 --- a/dmp-backend/src/main/java/proxy/config/UrlConfig.java +++ b/dmp-backend/src/main/java/proxy/config/UrlConfig.java @@ -1,23 +1,25 @@ package proxy.config; -import java.net.URL; import javax.xml.bind.annotation.XmlElement; - +//private static String DATAPATH = "$['data'][*]['attributes']['pid','name','uri','description']"; +//private static String PAGINATIONPATH = "$['meta']['pagination']['page','pages','count']"; public class UrlConfig { - private URL url; private Integer ordinal; + private String url; + private String dataPath; + private String paginationPath; - public URL getUrl() { + public String getUrl() { return url; } @XmlElement(name = "url") - public void setUrl(URL url) { + public void setUrl(String url) { this.url = url; } public Integer getOrdinal() { @@ -27,6 +29,22 @@ public class UrlConfig { public void setOrdinal(Integer ordinal) { this.ordinal = ordinal; } + + public String getDataPath() { + return dataPath; + } + @XmlElement(name = "datapath") + public void setDataPath(String dataPath) { + this.dataPath = dataPath; + } + + public String getPaginationPath() { + return paginationPath; + } + @XmlElement(name = "paginationpath") + public void setPaginationPath(String paginationPath) { + this.paginationPath = paginationPath; + } diff --git a/dmp-backend/src/main/java/proxy/config/entities/ProjectUrls.java b/dmp-backend/src/main/java/proxy/config/entities/ProjectUrls.java new file mode 100644 index 000000000..3f2b32bd8 --- /dev/null +++ b/dmp-backend/src/main/java/proxy/config/entities/ProjectUrls.java @@ -0,0 +1,33 @@ +package proxy.config.entities; + +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; + +import proxy.config.FetchStrategy; +import proxy.config.UrlConfig; + + +public class ProjectUrls { + + List urls; + FetchStrategy fetchMode; + + public List getUrls() { + return urls; + } + @XmlElementWrapper + @XmlElement(name = "urlConfig") + public void setUrls(List urls) { + this.urls = urls; + } + public FetchStrategy getFetchMode() { + return fetchMode; + } + @XmlElement(name = "fetchMode") + public void setFetchMode(FetchStrategy fetchMode) { + this.fetchMode = fetchMode; + } + +} \ No newline at end of file diff --git a/dmp-backend/src/main/java/proxy/config/exceptions/NoURLFound.java b/dmp-backend/src/main/java/proxy/config/exceptions/NoURLFound.java new file mode 100644 index 000000000..c0c407882 --- /dev/null +++ b/dmp-backend/src/main/java/proxy/config/exceptions/NoURLFound.java @@ -0,0 +1,13 @@ +package proxy.config.exceptions; + +public class NoURLFound extends Exception { + + private static final long serialVersionUID = -6961447213733280563L; + + + public NoURLFound(String message) { + super(message); + } + + +} diff --git a/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java b/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java new file mode 100644 index 000000000..a64bfc0ff --- /dev/null +++ b/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java @@ -0,0 +1,196 @@ +package proxy.fetching; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.Cacheable; + +import com.google.common.collect.ContiguousSet; +import com.google.common.collect.DiscreteDomain; +import com.google.common.collect.Range; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; + +import proxy.config.ConfigLoader; +import proxy.config.UrlConfig; +import proxy.config.exceptions.NoURLFound; + +public class RemoteFetcher { + + + @Autowired private ConfigLoader configLoader; + + +// public static void main(String [] args) throws Exception { +// +// String path = "https://eestore.paas2.uninett.no/api/datarepo"; +// String query = "her"; +// +// RemoteFetcher remoteFetcher = new RemoteFetcher(); +// List> repos = remoteFetcher.getAllResultsFromUrl(path, query); +// +//// List> repos = remoteFetcher.getRepositories(query); +// System.out.println(repos.size()); +// +// } + + @Cacheable("repositories") + public List> getRepositories(String query) throws NoURLFound { + List urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls(); + return getAll(urlConfigs, query); + } + + @Cacheable("projects") + public List> getProjects(String query) throws NoURLFound { + List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); + return getAll(urlConfigs, query); + } + + @Cacheable("organisations") + public List> getOrganisations(String query) throws NoURLFound { + List urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls(); + return getAll(urlConfigs, query); + } + + @Cacheable("registries") + public List> getRegistries(String query) throws NoURLFound { + List urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls(); + return getAll(urlConfigs, query); + } + + @Cacheable("services") + public List> getServices(String query) throws NoURLFound { + List urlConfigs = configLoader.getExternalUrls().getServices().getUrls(); + return getAll(urlConfigs, query); + } + + @Cacheable("researchers") + public List> getResearchers(String query) throws NoURLFound { + List urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls(); + return getAll(urlConfigs, query); + } + + + + + private List> getAll(List urlConfigs, String query) throws NoURLFound{ + + if(urlConfigs == null || urlConfigs.isEmpty()) + throw new NoURLFound("No Repository urls found in configuration"); + + Collections.sort(urlConfigs, (config1, config2) -> config1.getOrdinal().compareTo(config2.getOrdinal())); + + //for the time being, we only get the first one. in the near future we can add more than one (parallel stream) + //urlConfigs.parallelStream().map(mapper).reduce() etc etc + + return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query); + + } + + + + + + private List> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) { + Set pages = new HashSet(); + + final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : ""; + + //first call + Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath); + + if(results.getPagination()!= null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set + for(int i = 2; i <= results.getPagination().get("pages") ; i++) + pages.add(i); + + //remaining calls (if pages array has elements) + Optional optionalResults = pages.parallelStream() + .map(page -> getResultsFromUrl(path + "?page="+page + searchQuery, jsonDataPath, jsonPaginationPath)) + .reduce((result1, result2) -> { + result1.getResults().addAll(result2.getResults()); + return result1; + }); + Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results(); + + remainingResults.getResults().addAll(results.getResults()); + + return remainingResults.getResults(); + } + + + private Results getResultsFromUrl(String urlString, String jsonDataPath, String jsonPaginationPath) { + + try { + + URL url = new URL(urlString); + + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8"); + + int responseCode = con.getResponseCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { // success + //do here all the parsing + DocumentContext jsonContext = JsonPath.parse(con.getInputStream()); + Results results = new Results(jsonContext.read(jsonDataPath), jsonContext.read(jsonPaginationPath)); + return results; + } + } + catch(MalformedURLException e1) {} //maybe print smth... + catch(IOException e2) {} //maybe print smth... + finally {} + + return null; + + } + + + + + class Results { + + List> results; + Map pagination; + + public Results() { + this.results = new ArrayList>(); + this.pagination = new HashMap(); + } + + public Results(List> results, Map pagination) { + this.results = results; + this.pagination = pagination; + } + + public List> getResults() { + return results; + } + public void setResults(List> results) { + this.results = results; + } + public Map getPagination() { + return pagination; + } + public void setPagination(Map pagination) { + this.pagination = pagination; + } + + + + + } + +} diff --git a/dmp-backend/src/main/java/rest/entities/DataRepositories.java b/dmp-backend/src/main/java/rest/entities/DataRepositories.java index bb016998e..7b6355d27 100644 --- a/dmp-backend/src/main/java/rest/entities/DataRepositories.java +++ b/dmp-backend/src/main/java/rest/entities/DataRepositories.java @@ -1,6 +1,7 @@ package rest.entities; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -36,6 +37,7 @@ import dao.entities.ProjectDao; import dao.entities.RegistryDao; import dao.entities.ResearcherDao; import dao.entities.ServiceDao; +import dao.entities.UserInfoDao; import entities.DMP; import entities.DMPProfile; import entities.DataRepository; @@ -49,6 +51,7 @@ import entities.Researcher; import entities.Service; import helpers.SerializerProvider; import helpers.Transformers; +import proxy.fetching.RemoteFetcher; import responses.RestResponse; @@ -69,6 +72,26 @@ public class DataRepositories { @Autowired private ResearcherDao researcherDao; @Autowired private ServiceDao serviceDao; + @Autowired private RemoteFetcher remoteFetcher; + + + + + + @RequestMapping(method = RequestMethod.GET, value = { "/externaldatarepo" }, produces="application/json") + public @ResponseBody ResponseEntity listExternalDataRepositories(@RequestParam("query") String query ){ + try { + List> remoteRepos = remoteFetcher.getRepositories(query); + return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos)); + } + catch(Exception ex) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Serialization issue: "+ex.getMessage()); + } + } + + + + // MANAGE DATAREPOSITORy(IES) @RequestMapping(method = RequestMethod.GET, value = { "/datarepos" }) diff --git a/dmp-backend/src/main/resources/ExternalUrls.xml b/dmp-backend/src/main/resources/ExternalUrls.xml index 5dba8d692..94e9f125d 100644 --- a/dmp-backend/src/main/resources/ExternalUrls.xml +++ b/dmp-backend/src/main/resources/ExternalUrls.xml @@ -6,27 +6,48 @@ - - 1 - - - - ALL - - - - - - - - - https://eestore.paas2.uninett.no/api/datarepo/ 1 + https://eestore.paas2.uninett.no/api/metadataschemarepo/ + $['data'][*]['attributes']['pid','name','uri','description'] + $['meta']['pagination']['page','pages','count'] ALL + + + + + + + + 1 + https://eestore.paas2.uninett.no/api/projectrepo/ + $['data'][*]['attributes']['pid','name','uri','description'] + $['meta']['pagination']['page','pages','count'] + + + + + ALL + + + + + + + + 1 + https://eestore.paas2.uninett.no/api/datarepo/ + $['data'][*]['attributes']['pid','name','uri','description'] + $['meta']['pagination']['page','pages','count'] + + + + + ALL + @@ -34,9 +55,12 @@ - 1 + https://eestore.paas2.uninett.no/api/servicerepo/ + $['data'][*]['attributes']['pid','name','uri','description'] + $['meta']['pagination']['page','pages','count'] + ALL @@ -47,9 +71,12 @@ - 1 + https://eestore.paas2.uninett.no/api/personrepo/ + $['data'][*]['attributes']['pid','name','uri','description'] + $['meta']['pagination']['page','pages','count'] + ALL @@ -61,9 +88,12 @@ - 1 + https://eestore.paas2.uninett.no/api/organizationrepo/ + $['data'][*]['attributes']['pid','name','uri','description'] + $['meta']['pagination']['page','pages','count'] + ALL diff --git a/dmp-backend/src/main/webapp/WEB-INF/applicationContext.xml b/dmp-backend/src/main/webapp/WEB-INF/applicationContext.xml index dcbfe3751..09525e509 100644 --- a/dmp-backend/src/main/webapp/WEB-INF/applicationContext.xml +++ b/dmp-backend/src/main/webapp/WEB-INF/applicationContext.xml @@ -1,16 +1,21 @@ + http://www.springframework.org/schema/context/spring-context.xsd + http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> + + @@ -44,9 +49,12 @@ - + + + + diff --git a/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-login-servlet.xml b/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-login-servlet.xml index b09d16973..de5ce7fcb 100644 --- a/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-login-servlet.xml +++ b/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-login-servlet.xml @@ -1,13 +1,16 @@ + http://www.springframework.org/schema/beans/spring-beans-4.1.xsd + http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> @@ -16,6 +19,8 @@ + + diff --git a/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-proxy-servlet.xml b/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-proxy-servlet.xml index a24cbb4af..91c777e61 100644 --- a/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-proxy-servlet.xml +++ b/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-proxy-servlet.xml @@ -1,13 +1,16 @@ + http://www.springframework.org/schema/beans/spring-beans-4.1.xsd + http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> @@ -16,6 +19,7 @@ + diff --git a/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-rest-servlet.xml b/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-rest-servlet.xml index 5bce8017b..ffa8bc397 100644 --- a/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-rest-servlet.xml +++ b/dmp-backend/src/main/webapp/WEB-INF/dmp-backend-rest-servlet.xml @@ -1,22 +1,30 @@ + http://www.springframework.org/schema/beans/spring-beans-4.1.xsd + http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> - - + + + + + + + diff --git a/dmp-backend/src/test/resources/ExternalUrlsTEST.xml b/dmp-backend/src/test/resources/ExternalUrlsTEST.xml new file mode 100644 index 000000000..5dba8d692 --- /dev/null +++ b/dmp-backend/src/test/resources/ExternalUrlsTEST.xml @@ -0,0 +1,80 @@ + + + + + + + + + + 1 + + + + ALL + + + + + + + + + https://eestore.paas2.uninett.no/api/datarepo/ + 1 + + + + + ALL + + + + + + + + + 1 + + + + ALL + + + + + + + + + 1 + + + + ALL + + + + + + + + + + 1 + + + + ALL + + + + + + + + + + + diff --git a/dmp-backend/src/test/resources/applicationContextTEST.xml b/dmp-backend/src/test/resources/applicationContextTEST.xml index fced92d12..91f49658e 100644 --- a/dmp-backend/src/test/resources/applicationContextTEST.xml +++ b/dmp-backend/src/test/resources/applicationContextTEST.xml @@ -35,6 +35,12 @@ + + + + + + @@ -112,6 +118,7 @@ +