MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Switched to "first" as default fetching strategy

This commit is contained in:
Nikolaos Laskaris 2017-12-06 17:22:45 +02:00
parent fe6ab0d125
commit 5cb2d081c5
2 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java
View File

@ -26,7 +26,9 @@ import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import proxy.config.ConfigLoader;
import proxy.config.FetchStrategy;
import proxy.config.UrlConfig;
import proxy.config.entities.ServiceUrls;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
@ -39,59 +41,52 @@ public class RemoteFetcher {
// private static int MAX_RESULTS = 30;
// public static void main(String [] args) throws Exception {
//
// String path = "https://eestore.paas2.uninett.no/api/datarepo";
// String query = "her";
//
// RemoteFetcher remoteFetcher = new RemoteFetcher();
// List<Map<String, String>> repos = remoteFetcher.getAllResultsFromUrl(path, query);
//
//// List<Map<String, String>> repos = remoteFetcher.getRepositories(query);
// System.out.println(repos.size());
//
// }
@Cacheable("repositories")
public List<Map<String, String>> getRepositories(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls();
return getAll(urlConfigs, query);
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("projects")
public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
return getAll(urlConfigs, query);
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("organisations")
public List<Map<String, String>> getOrganisations(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls();
return getAll(urlConfigs, query);
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("registries")
public List<Map<String, String>> getRegistries(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls();
return getAll(urlConfigs, query);
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("services")
public List<Map<String, String>> getServices(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getServices().getUrls();
return getAll(urlConfigs, query);
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("researchers")
public List<Map<String, String>> getResearchers(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls();
return getAll(urlConfigs, query);
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, String query) throws NoURLFound, HugeResultSet{
private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet{
if(urlConfigs == null || urlConfigs.isEmpty())
throw new NoURLFound("No Repository urls found in configuration");
@ -101,7 +96,7 @@ public class RemoteFetcher {
//for the time being, we only get the first one. in the near future we can add more than one (parallel stream)
//urlConfigs.parallelStream().map(mapper).reduce() etc etc
return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query);
return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), fetchStrategy, urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query);
}
@ -109,7 +104,7 @@ public class RemoteFetcher {
private List<Map<String, String>> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet {
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet {
Set<Integer> pages = new HashSet<Integer>();
final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : "";
@ -117,6 +112,10 @@ public class RemoteFetcher {
//first call
Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath);
//if fetch strategy is to get only first page, then return that
if(fetchStrategy == FetchStrategy.FIRST)
return results.getResults();
if(results.getPagination()!= null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for(int i = 2; i <= results.getPagination().get("pages") ; i++)
pages.add(i);

12
dmp-backend/src/main/resources/ExternalUrls.xml
View File

@ -15,7 +15,7 @@
</urls>
<fetchMode>ALL</fetchMode>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</registries>
@ -31,7 +31,7 @@
</urls>
<fetchMode>ALL</fetchMode>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</projects>
@ -47,7 +47,7 @@
</urls>
<fetchMode>ALL</fetchMode>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</repositories>
@ -64,7 +64,7 @@
</urls>
<fetchMode>ALL</fetchMode>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</services>
@ -80,7 +80,7 @@
</urls>
<fetchMode>ALL</fetchMode>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</researchers>
@ -97,7 +97,7 @@
</urls>
<fetchMode>ALL</fetchMode>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</organisations>