2017-11-21 17:29:16 +01:00
|
|
|
package proxy.fetching;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.net.HttpURLConnection;
|
|
|
|
import java.net.MalformedURLException;
|
|
|
|
import java.net.URL;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.HashSet;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Optional;
|
|
|
|
import java.util.Set;
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
import java.util.stream.IntStream;
|
|
|
|
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.cache.annotation.Cacheable;
|
2017-11-22 09:57:51 +01:00
|
|
|
import org.springframework.stereotype.Service;
|
2017-11-21 17:29:16 +01:00
|
|
|
|
|
|
|
import com.google.common.collect.ContiguousSet;
|
|
|
|
import com.google.common.collect.DiscreteDomain;
|
|
|
|
import com.google.common.collect.Range;
|
|
|
|
import com.jayway.jsonpath.DocumentContext;
|
|
|
|
import com.jayway.jsonpath.JsonPath;
|
|
|
|
|
|
|
|
import proxy.config.ConfigLoader;
|
|
|
|
import proxy.config.UrlConfig;
|
|
|
|
import proxy.config.exceptions.NoURLFound;
|
|
|
|
|
2017-11-22 09:57:51 +01:00
|
|
|
@Service
|
2017-11-21 17:29:16 +01:00
|
|
|
public class RemoteFetcher {
|
|
|
|
|
|
|
|
|
|
|
|
@Autowired private ConfigLoader configLoader;
|
|
|
|
|
|
|
|
|
|
|
|
// public static void main(String [] args) throws Exception {
|
|
|
|
//
|
|
|
|
// String path = "https://eestore.paas2.uninett.no/api/datarepo";
|
|
|
|
// String query = "her";
|
|
|
|
//
|
|
|
|
// RemoteFetcher remoteFetcher = new RemoteFetcher();
|
|
|
|
// List<Map<String, String>> repos = remoteFetcher.getAllResultsFromUrl(path, query);
|
|
|
|
//
|
|
|
|
//// List<Map<String, String>> repos = remoteFetcher.getRepositories(query);
|
|
|
|
// System.out.println(repos.size());
|
|
|
|
//
|
|
|
|
// }
|
|
|
|
|
|
|
|
@Cacheable("repositories")
|
|
|
|
public List<Map<String, String>> getRepositories(String query) throws NoURLFound {
|
2017-11-22 09:57:51 +01:00
|
|
|
System.out.println("NOT CACHED");
|
2017-11-21 17:29:16 +01:00
|
|
|
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls();
|
|
|
|
return getAll(urlConfigs, query);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Cacheable("projects")
|
|
|
|
public List<Map<String, String>> getProjects(String query) throws NoURLFound {
|
|
|
|
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
|
|
|
|
return getAll(urlConfigs, query);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Cacheable("organisations")
|
|
|
|
public List<Map<String, String>> getOrganisations(String query) throws NoURLFound {
|
|
|
|
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls();
|
|
|
|
return getAll(urlConfigs, query);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Cacheable("registries")
|
|
|
|
public List<Map<String, String>> getRegistries(String query) throws NoURLFound {
|
|
|
|
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls();
|
|
|
|
return getAll(urlConfigs, query);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Cacheable("services")
|
|
|
|
public List<Map<String, String>> getServices(String query) throws NoURLFound {
|
|
|
|
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getServices().getUrls();
|
|
|
|
return getAll(urlConfigs, query);
|
|
|
|
}
|
|
|
|
|
|
|
|
@Cacheable("researchers")
|
|
|
|
public List<Map<String, String>> getResearchers(String query) throws NoURLFound {
|
|
|
|
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls();
|
|
|
|
return getAll(urlConfigs, query);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, String query) throws NoURLFound{
|
|
|
|
|
|
|
|
if(urlConfigs == null || urlConfigs.isEmpty())
|
|
|
|
throw new NoURLFound("No Repository urls found in configuration");
|
|
|
|
|
|
|
|
Collections.sort(urlConfigs, (config1, config2) -> config1.getOrdinal().compareTo(config2.getOrdinal()));
|
|
|
|
|
|
|
|
//for the time being, we only get the first one. in the near future we can add more than one (parallel stream)
|
|
|
|
//urlConfigs.parallelStream().map(mapper).reduce() etc etc
|
|
|
|
|
|
|
|
return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private List<Map<String, String>> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) {
|
|
|
|
Set<Integer> pages = new HashSet<Integer>();
|
|
|
|
|
|
|
|
final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : "";
|
|
|
|
|
|
|
|
//first call
|
|
|
|
Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath);
|
|
|
|
|
|
|
|
if(results.getPagination()!= null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
|
|
|
|
for(int i = 2; i <= results.getPagination().get("pages") ; i++)
|
|
|
|
pages.add(i);
|
|
|
|
|
|
|
|
//remaining calls (if pages array has elements)
|
|
|
|
Optional<Results> optionalResults = pages.parallelStream()
|
|
|
|
.map(page -> getResultsFromUrl(path + "?page="+page + searchQuery, jsonDataPath, jsonPaginationPath))
|
|
|
|
.reduce((result1, result2) -> {
|
|
|
|
result1.getResults().addAll(result2.getResults());
|
|
|
|
return result1;
|
|
|
|
});
|
|
|
|
Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results();
|
|
|
|
|
|
|
|
remainingResults.getResults().addAll(results.getResults());
|
|
|
|
|
|
|
|
return remainingResults.getResults();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private Results getResultsFromUrl(String urlString, String jsonDataPath, String jsonPaginationPath) {
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
|
|
|
URL url = new URL(urlString);
|
|
|
|
|
|
|
|
HttpURLConnection con = (HttpURLConnection) url.openConnection();
|
|
|
|
con.setRequestMethod("GET");
|
|
|
|
con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8");
|
|
|
|
|
|
|
|
int responseCode = con.getResponseCode();
|
|
|
|
if (responseCode == HttpURLConnection.HTTP_OK) { // success
|
|
|
|
//do here all the parsing
|
|
|
|
DocumentContext jsonContext = JsonPath.parse(con.getInputStream());
|
|
|
|
Results results = new Results(jsonContext.read(jsonDataPath), jsonContext.read(jsonPaginationPath));
|
|
|
|
return results;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
catch(MalformedURLException e1) {} //maybe print smth...
|
|
|
|
catch(IOException e2) {} //maybe print smth...
|
|
|
|
finally {}
|
|
|
|
|
|
|
|
return null;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Results {
|
|
|
|
|
|
|
|
List<Map<String, String>> results;
|
|
|
|
Map<String, Integer> pagination;
|
|
|
|
|
|
|
|
public Results() {
|
|
|
|
this.results = new ArrayList<Map<String,String>>();
|
|
|
|
this.pagination = new HashMap<String,Integer>();
|
|
|
|
}
|
|
|
|
|
|
|
|
public Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
|
|
|
|
this.results = results;
|
|
|
|
this.pagination = pagination;
|
|
|
|
}
|
|
|
|
|
|
|
|
public List<Map<String, String>> getResults() {
|
|
|
|
return results;
|
|
|
|
}
|
|
|
|
public void setResults(List<Map<String, String>> results) {
|
|
|
|
this.results = results;
|
|
|
|
}
|
|
|
|
public Map<String, Integer> getPagination() {
|
|
|
|
return pagination;
|
|
|
|
}
|
|
|
|
public void setPagination(Map<String, Integer> pagination) {
|
|
|
|
this.pagination = pagination;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|