argos/dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java

package proxy.fetching;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;

import com.google.common.collect.ContiguousSet;
import com.google.common.collect.DiscreteDomain;
import com.google.common.collect.Range;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;

import proxy.config.ConfigLoader;
import proxy.config.UrlConfig;
import proxy.config.exceptions.NoURLFound;

public class RemoteFetcher {
	
	
	@Autowired private ConfigLoader configLoader;
	
	
//	public static void main(String [] args) throws Exception {
//		
//		String path = "https://eestore.paas2.uninett.no/api/datarepo";
//		String query = "her";
//		
//		RemoteFetcher remoteFetcher = new RemoteFetcher();
//		List<Map<String, String>> repos = remoteFetcher.getAllResultsFromUrl(path, query);
//		
////		List<Map<String, String>> repos = remoteFetcher.getRepositories(query);
//		System.out.println(repos.size());
//		
//	}
	
	@Cacheable("repositories")
	public List<Map<String, String>> getRepositories(String query) throws NoURLFound {
		List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls();
		return getAll(urlConfigs, query);
	}
	
	@Cacheable("projects")
	public List<Map<String, String>> getProjects(String query) throws NoURLFound {
		List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
		return getAll(urlConfigs, query);
	}
	
	@Cacheable("organisations")
	public List<Map<String, String>> getOrganisations(String query) throws NoURLFound {
		List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls();
		return getAll(urlConfigs, query);
	}
	
	@Cacheable("registries")
	public List<Map<String, String>> getRegistries(String query) throws NoURLFound {
		List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls();
		return getAll(urlConfigs, query);
	}
	
	@Cacheable("services")
	public List<Map<String, String>> getServices(String query) throws NoURLFound {
		List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getServices().getUrls();
		return getAll(urlConfigs, query);
	}
	
	@Cacheable("researchers")
	public List<Map<String, String>> getResearchers(String query) throws NoURLFound {
		List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls();
		return getAll(urlConfigs, query);
	}
	
	
	
	
	private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, String query) throws NoURLFound{
		
		if(urlConfigs == null || urlConfigs.isEmpty())
			throw new NoURLFound("No Repository urls found in configuration");
		
		Collections.sort(urlConfigs, (config1, config2) -> config1.getOrdinal().compareTo(config2.getOrdinal()));
		
		//for the time being, we only get the first one. in the near future we can add more than one (parallel stream)
		//urlConfigs.parallelStream().map(mapper).reduce() etc etc
		
		return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query);
		
	}
	
	
	
	
	
	private List<Map<String, String>> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) {
		Set<Integer> pages = new HashSet<Integer>();
		
		final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : "";
		
		//first call
		Results results = getResultsFromUrl(path + "?page=1" + searchQuery,  jsonDataPath, jsonPaginationPath);
		
		if(results.getPagination()!= null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
			for(int i = 2; i <= results.getPagination().get("pages") ; i++)
				pages.add(i);
		
		//remaining calls (if pages array has elements)
		Optional<Results> optionalResults = pages.parallelStream()
		.map(page -> getResultsFromUrl(path + "?page="+page + searchQuery, jsonDataPath, jsonPaginationPath))
		.reduce((result1, result2) -> {
			result1.getResults().addAll(result2.getResults());
			return result1;
		});
		Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results();
		
		remainingResults.getResults().addAll(results.getResults());
		
		return remainingResults.getResults();
	}
	
	
	private Results getResultsFromUrl(String urlString, String jsonDataPath, String jsonPaginationPath) {
		
		try {
		
			URL url = new URL(urlString);
			
			HttpURLConnection con = (HttpURLConnection) url.openConnection();
			con.setRequestMethod("GET");
			con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8");
			
			int responseCode = con.getResponseCode();
			if (responseCode == HttpURLConnection.HTTP_OK) { // success
				//do here all the parsing
				DocumentContext jsonContext = JsonPath.parse(con.getInputStream());
				Results results = new Results(jsonContext.read(jsonDataPath), jsonContext.read(jsonPaginationPath));
				return results;
			} 
		}
		catch(MalformedURLException e1) {} //maybe print smth...
		catch(IOException e2) {} //maybe print smth...
		finally {}
		
		return null;
		
	}
	
	
	
	
	class Results {
		
		List<Map<String, String>> results;
		Map<String, Integer> pagination;
		
		public Results() {
			this.results = new ArrayList<Map<String,String>>();
			this.pagination = new HashMap<String,Integer>();
		}
		
		public Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
			this.results = results;
			this.pagination = pagination;
		}
		
		public List<Map<String, String>> getResults() {
			return results;
		}
		public void setResults(List<Map<String, String>> results) {
			this.results = results;
		}
		public Map<String, Integer> getPagination() {
			return pagination;
		}
		public void setPagination(Map<String, Integer> pagination) {
			this.pagination = pagination;
		}
		
		
		
		
	}

}