297 lines
15 KiB
Java
297 lines
15 KiB
Java
package eu.eudat.logic.proxy.fetching;
|
|
|
|
import com.fasterxml.jackson.core.type.TypeReference;
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.jayway.jsonpath.DocumentContext;
|
|
import com.jayway.jsonpath.JsonPath;
|
|
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
|
|
import eu.eudat.logic.proxy.config.FetchStrategy;
|
|
import eu.eudat.logic.proxy.config.UrlConfiguration;
|
|
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
|
|
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
|
|
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.cache.annotation.Cacheable;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URL;
|
|
import java.util.*;
|
|
import java.util.stream.Collectors;
|
|
|
|
@Service
|
|
public class RemoteFetcher {
|
|
|
|
private ConfigLoader configLoader;
|
|
|
|
@Value("${configuration.resources.path}")
|
|
private String resourcesPath;
|
|
|
|
@Autowired
|
|
public RemoteFetcher(ConfigLoader configLoader) {
|
|
this.configLoader = configLoader;
|
|
}
|
|
|
|
@Cacheable("repositories")
|
|
public List<Map<String, String>> getRepositories(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getRepositories().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("grants")
|
|
public List<Map<String, String>> getGrants(String query) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("projects")
|
|
public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("funders")
|
|
public List<Map<String, String>> getFunders(String query) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("organisations")
|
|
public List<Map<String, String>> getOrganisations(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getOrganisations().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("registries")
|
|
public List<Map<String, String>> getRegistries(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getRegistries().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("services")
|
|
public List<Map<String, String>> getServices(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getServices().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("researchers")
|
|
public List<Map<String, String>> getResearchers(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getResearchers().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("tags")
|
|
public List<Map<String, String>> getTags(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getTags().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
@Cacheable("datasets")
|
|
public List<Map<String, String>> getDatasets(String query, String key) throws NoURLFound, HugeResultSet {
|
|
List<UrlConfiguration> urlConfigs =
|
|
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
|
: configLoader.getExternalUrls().getDatasets().getUrls();
|
|
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode();
|
|
return getAll(urlConfigs, fetchStrategy, query);
|
|
}
|
|
|
|
|
|
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet {
|
|
|
|
if (urlConfigs == null || urlConfigs.isEmpty())
|
|
throw new NoURLFound("No Repository urls found in configuration");
|
|
|
|
Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal));
|
|
List<Map<String, String>> results = new LinkedList<>();
|
|
for (UrlConfiguration urlConfig : urlConfigs) {
|
|
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
|
|
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel()));
|
|
}
|
|
else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
|
|
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query));
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
|
|
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key) throws HugeResultSet {
|
|
Set<Integer> pages = new HashSet<Integer>();
|
|
|
|
final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : "";
|
|
|
|
Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath);
|
|
|
|
if (fetchStrategy == FetchStrategy.FIRST)
|
|
return results == null ? new LinkedList<>() : results.getResults().stream().map(x -> {
|
|
x.put("tag", key);
|
|
return x;
|
|
}).collect(Collectors.toList());
|
|
|
|
if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
|
|
for (int i = 2; i <= results.getPagination().get("pages"); i++)
|
|
pages.add(i);
|
|
|
|
Long maxResults = configLoader.getExternalUrls().getMaxresults();
|
|
if ((maxResults > 0) && (results.getPagination().get("count") > maxResults))
|
|
throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
|
|
|
|
Optional<Results> optionalResults = pages.parallelStream()
|
|
.map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath))
|
|
.reduce((result1, result2) -> {
|
|
result1.getResults().addAll(result2.getResults());
|
|
return result1;
|
|
});
|
|
Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results();
|
|
|
|
remainingResults.getResults().addAll(results.getResults());
|
|
|
|
return remainingResults.getResults().stream().map(x -> {
|
|
x.put("tag", key);
|
|
return x;
|
|
}).collect(Collectors.toList());
|
|
}
|
|
|
|
|
|
private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath) {
|
|
|
|
try {
|
|
|
|
URL url = new URL(urlString);
|
|
|
|
HttpURLConnection con = (HttpURLConnection) url.openConnection();
|
|
con.setRequestMethod("GET");
|
|
con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8");
|
|
|
|
int responseCode = con.getResponseCode();
|
|
if (responseCode == HttpURLConnection.HTTP_OK) { // success
|
|
//do here all the parsing
|
|
DocumentContext jsonContext = JsonPath.parse(con.getInputStream());
|
|
Results results;
|
|
if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) {
|
|
results = new Results(jsonContext.read(jsonDataPath.getPath()
|
|
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
|
|
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
|
|
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
|
|
jsonContext.read(jsonPaginationPath));
|
|
}
|
|
else {
|
|
results = new Results(jsonContext.read(jsonDataPath.getPath()
|
|
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
|
|
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
|
|
jsonContext.read(jsonPaginationPath));
|
|
}
|
|
results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
|
|
.collect(Collectors.toList());
|
|
return results;
|
|
}
|
|
} catch (MalformedURLException e1) {
|
|
e1.printStackTrace();
|
|
} //maybe print smth...
|
|
catch (IOException e2) {
|
|
e2.printStackTrace();
|
|
} //maybe print smth...
|
|
catch (Exception exception) {
|
|
exception.printStackTrace();
|
|
} //maybe print smth...
|
|
finally {
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
|
|
String filePath = this.resourcesPath + path;
|
|
List<Map<String, String>> internalResults;
|
|
try {
|
|
ObjectMapper mapper = new ObjectMapper();
|
|
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, Object>>>(){});
|
|
searchListMap(internalResults, query);
|
|
return internalResults;
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
return new LinkedList<>();
|
|
}
|
|
}
|
|
|
|
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
|
|
List<Map<String, String>> list = new LinkedList<>();
|
|
for (Map<String, String> map : internalResults)
|
|
{
|
|
if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
|
|
list.add(map);
|
|
}
|
|
if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
|
|
list.add(map);
|
|
}
|
|
}
|
|
return list;
|
|
}
|
|
|
|
private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
|
|
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
|
|
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
|
|
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
|
|
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
|
|
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
|
|
return null;
|
|
}
|
|
|
|
|
|
static class Results {
|
|
List<Map<String, String>> results;
|
|
Map<String, Integer> pagination;
|
|
|
|
Results() {
|
|
this.results = new ArrayList<Map<String, String>>();
|
|
this.pagination = new HashMap<String, Integer>();
|
|
}
|
|
|
|
Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
|
|
this.results = results;
|
|
this.pagination = pagination;
|
|
}
|
|
|
|
List<Map<String, String>> getResults() {
|
|
return results;
|
|
}
|
|
|
|
public void setResults(List<Map<String, String>> results) {
|
|
this.results = results;
|
|
}
|
|
|
|
Map<String, Integer> getPagination() {
|
|
return pagination;
|
|
}
|
|
|
|
public void setPagination(Map<String, Integer> pagination) {
|
|
this.pagination = pagination;
|
|
}
|
|
}
|
|
}
|