argos/dmp-backend/web/src/main/java/eu/eudat/proxy/fetching/RemoteFetcher.java

package eu.eudat.proxy.fetching;

import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.dynamicproject.DynamicProjectConfiguration;
import eu.eudat.proxy.config.FetchStrategy;
import eu.eudat.proxy.config.UrlConfig;
import eu.eudat.proxy.config.configloaders.ConfigLoader;
import eu.eudat.proxy.config.exceptions.HugeResultSet;
import eu.eudat.proxy.config.exceptions.NoURLFound;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;

@Service
public class RemoteFetcher {

    private ConfigLoader configLoader;

    private DynamicProjectConfiguration dynamicProjectConfiguration;

    @Autowired
    public RemoteFetcher(ConfigLoader configLoader, DynamicProjectConfiguration dynamicProjectConfiguration) {
        this.configLoader = configLoader;
        this.dynamicProjectConfiguration = dynamicProjectConfiguration;
    }

    @Cacheable("repositories")
    public List<Map<String, String>> getRepositories(String query, String key) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs =
                key != null ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                        : configLoader.getExternalUrls().getRepositories().getUrls();
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }

    @Cacheable("projects")
    public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs = Arrays.asList(this.dynamicProjectConfiguration.getConfiguration().getMainExternalField().getUrlConfig());
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }

    @Cacheable("organisations")
    public List<Map<String, String>> getOrganisations(String query, String key) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs =
                key != null ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                        : configLoader.getExternalUrls().getOrganisations().getUrls();
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }

    @Cacheable("registries")
    public List<Map<String, String>> getRegistries(String query, String key) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs =
                key != null ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                        : configLoader.getExternalUrls().getRegistries().getUrls();
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }

    @Cacheable("services")
    public List<Map<String, String>> getServices(String query,String key) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs =
                key != null ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                        : configLoader.getExternalUrls().getServices().getUrls();
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }

    @Cacheable("researchers")
    public List<Map<String, String>> getResearchers(String query,String key) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs =
                key != null ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                        : configLoader.getExternalUrls().getResearchers().getUrls();
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }

    @Cacheable("datasets")
    public List<Map<String, String>> getDatasets(String query,String key) throws NoURLFound, HugeResultSet {
        List<UrlConfig> urlConfigs =
                key != null ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
                        : configLoader.getExternalUrls().getDatasets().getUrls();
        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode();
        return getAll(urlConfigs, fetchStrategy, query);
    }


    private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet {

        if (urlConfigs == null || urlConfigs.isEmpty())
            throw new NoURLFound("No Repository urls found in configuration");

        Collections.sort(urlConfigs, Comparator.comparing(UrlConfig::getOrdinal));

        return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), fetchStrategy, urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query);

    }


    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet {
        Set<Integer> pages = new HashSet<Integer>();

        final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : "";

        //first call
        Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath);

        //if fetch strategy is to get only first page, then return that
        if (fetchStrategy == FetchStrategy.FIRST)
            return results == null ? new LinkedList<>() : results.getResults();

        if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
            for (int i = 2; i <= results.getPagination().get("pages"); i++)
                pages.add(i);

        Long maxResults = configLoader.getExternalUrls().getMaxresults();
        if ((maxResults > 0) && (results.getPagination().get("count") > maxResults))
            throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");

        //remaining calls (if pages array has elements)
        Optional<Results> optionalResults = pages.parallelStream()
                .map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath))
                .reduce((result1, result2) -> {
                    result1.getResults().addAll(result2.getResults());
                    return result1;
                });
        Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results();

        remainingResults.getResults().addAll(results.getResults());

        return remainingResults.getResults();
    }


    private Results getResultsFromUrl(String urlString, String jsonDataPath, String jsonPaginationPath) {

        try {

            URL url = new URL(urlString);

            HttpURLConnection con = (HttpURLConnection) url.openConnection();
            con.setRequestMethod("GET");
            con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8");

            int responseCode = con.getResponseCode();
            if (responseCode == HttpURLConnection.HTTP_OK) { // success
                //do here all the parsing
                DocumentContext jsonContext = JsonPath.parse(con.getInputStream());
                Results results = new Results(jsonContext.read(jsonDataPath), jsonContext.read(jsonPaginationPath));
                return results;
            }
        } catch (MalformedURLException e1) {
        } //maybe print smth...
        catch (IOException e2) {
        } //maybe print smth...
        finally {
        }

        return null;

    }


    class Results {

        List<Map<String, String>> results;
        Map<String, Integer> pagination;

        public Results() {
            this.results = new ArrayList<Map<String, String>>();
            this.pagination = new HashMap<String, Integer>();
        }

        public Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
            this.results = results;
            this.pagination = pagination;
        }

        public List<Map<String, String>> getResults() {
            return results;
        }

        public void setResults(List<Map<String, String>> results) {
            this.results = results;
        }

        public Map<String, Integer> getPagination() {
            return pagination;
        }

        public void setPagination(Map<String, Integer> pagination) {
            this.pagination = pagination;
        }


    }

}