From 5cb2d081c57c3639d19966759d64c026eecd987f Mon Sep 17 00:00:00 2001 From: Nikolaos Laskaris Date: Wed, 6 Dec 2017 17:22:45 +0200 Subject: [PATCH] Switched to "first" as default fetching strategy --- .../java/proxy/fetching/RemoteFetcher.java | 43 +++++++++---------- .../src/main/resources/ExternalUrls.xml | 12 +++--- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java b/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java index 9b73ea5b5..741718649 100644 --- a/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java @@ -26,7 +26,9 @@ import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import proxy.config.ConfigLoader; +import proxy.config.FetchStrategy; import proxy.config.UrlConfig; +import proxy.config.entities.ServiceUrls; import proxy.config.exceptions.HugeResultSet; import proxy.config.exceptions.NoURLFound; @@ -39,59 +41,52 @@ public class RemoteFetcher { // private static int MAX_RESULTS = 30; -// public static void main(String [] args) throws Exception { -// -// String path = "https://eestore.paas2.uninett.no/api/datarepo"; -// String query = "her"; -// -// RemoteFetcher remoteFetcher = new RemoteFetcher(); -// List> repos = remoteFetcher.getAllResultsFromUrl(path, query); -// -//// List> repos = remoteFetcher.getRepositories(query); -// System.out.println(repos.size()); -// -// } - @Cacheable("repositories") public List> getRepositories(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls(); - return getAll(urlConfigs, query); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("projects") public List> getProjects(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); - return getAll(urlConfigs, query); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("organisations") public List> getOrganisations(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls(); - return getAll(urlConfigs, query); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("registries") public List> getRegistries(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls(); - return getAll(urlConfigs, query); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("services") public List> getServices(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getServices().getUrls(); - return getAll(urlConfigs, query); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("researchers") public List> getResearchers(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls(); - return getAll(urlConfigs, query); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); } - private List> getAll(List urlConfigs, String query) throws NoURLFound, HugeResultSet{ + private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet{ if(urlConfigs == null || urlConfigs.isEmpty()) throw new NoURLFound("No Repository urls found in configuration"); @@ -101,7 +96,7 @@ public class RemoteFetcher { //for the time being, we only get the first one. in the near future we can add more than one (parallel stream) //urlConfigs.parallelStream().map(mapper).reduce() etc etc - return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query); + return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), fetchStrategy, urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query); } @@ -109,7 +104,7 @@ public class RemoteFetcher { - private List> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet { + private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet { Set pages = new HashSet(); final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : ""; @@ -117,6 +112,10 @@ public class RemoteFetcher { //first call Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath); + //if fetch strategy is to get only first page, then return that + if(fetchStrategy == FetchStrategy.FIRST) + return results.getResults(); + if(results.getPagination()!= null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set for(int i = 2; i <= results.getPagination().get("pages") ; i++) pages.add(i); diff --git a/dmp-backend/src/main/resources/ExternalUrls.xml b/dmp-backend/src/main/resources/ExternalUrls.xml index ddf4f0f6d..fbd4c3608 100644 --- a/dmp-backend/src/main/resources/ExternalUrls.xml +++ b/dmp-backend/src/main/resources/ExternalUrls.xml @@ -15,7 +15,7 @@ - ALL + FIRST @@ -31,7 +31,7 @@ - ALL + FIRST @@ -47,7 +47,7 @@ - ALL + FIRST @@ -64,7 +64,7 @@ - ALL + FIRST @@ -80,7 +80,7 @@ - ALL + FIRST @@ -97,7 +97,7 @@ - ALL + FIRST