package eu.eudat.logic.proxy.fetching; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import eu.eudat.configurations.dynamicproject.DynamicProjectConfiguration; import eu.eudat.logic.proxy.config.DataUrlConfiguration; import eu.eudat.logic.proxy.config.FetchStrategy; import eu.eudat.logic.proxy.config.UrlConfiguration; import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.util.*; import java.util.stream.Collectors; @Service public class RemoteFetcher { private ConfigLoader configLoader; private DynamicProjectConfiguration dynamicProjectConfiguration; @Autowired public RemoteFetcher(ConfigLoader configLoader, DynamicProjectConfiguration dynamicProjectConfiguration) { this.configLoader = configLoader; this.dynamicProjectConfiguration = dynamicProjectConfiguration; } @Cacheable("repositories") public List> getRepositories(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getRepositories().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("projects") public List> getProjects(String query) throws NoURLFound, HugeResultSet { List urlConfigs = Arrays.asList(this.dynamicProjectConfiguration.getConfiguration().getMainExternalField().getUrlConfig()); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("organisations") public List> getOrganisations(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getOrganisations().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("registries") public List> getRegistries(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getRegistries().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("services") public List> getServices(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getServices().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("researchers") public List> getResearchers(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getResearchers().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("tags") public List> getTags(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getTags().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("datasets") public List> getDatasets(String query, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getDatasets().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet { if (urlConfigs == null || urlConfigs.isEmpty()) throw new NoURLFound("No Repository urls found in configuration"); Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal)); List> results = new LinkedList<>(); for (UrlConfiguration urlConfig : urlConfigs) { results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel())); } return results; } private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key) throws HugeResultSet { Set pages = new HashSet(); final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : ""; Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath); if (fetchStrategy == FetchStrategy.FIRST) return results == null ? new LinkedList<>() : results.getResults().stream().map(x -> { x.put("tag", key); return x; }).collect(Collectors.toList()); if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set for (int i = 2; i <= results.getPagination().get("pages"); i++) pages.add(i); Long maxResults = configLoader.getExternalUrls().getMaxresults(); if ((maxResults > 0) && (results.getPagination().get("count") > maxResults)) throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); Optional optionalResults = pages.parallelStream() .map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath)) .reduce((result1, result2) -> { result1.getResults().addAll(result2.getResults()); return result1; }); Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results(); remainingResults.getResults().addAll(results.getResults()); return remainingResults.getResults().stream().map(x -> { x.put("tag", key); return x; }).collect(Collectors.toList()); } private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath) { try { URL url = new URL(urlString); HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("GET"); con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8"); int responseCode = con.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { // success //do here all the parsing DocumentContext jsonContext = JsonPath.parse(con.getInputStream()); Results results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), jsonContext.read(jsonPaginationPath)); results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) .collect(Collectors.toList()); return results; } } catch (MalformedURLException e1) { e1.printStackTrace(); } //maybe print smth... catch (IOException e2) { e2.printStackTrace(); } //maybe print smth... catch (Exception exception) { exception.printStackTrace(); } //maybe print smth... finally { } return null; } private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) { if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid"; if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description"; if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; return null; } class Results { List> results; Map pagination; public Results() { this.results = new ArrayList>(); this.pagination = new HashMap(); } public Results(List> results, Map pagination) { this.results = results; this.pagination = pagination; } public List> getResults() { return results; } public void setResults(List> results) { this.results = results; } public Map getPagination() { return pagination; } public void setPagination(Map pagination) { this.pagination = pagination; } } }