MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Added a max-results limit on external repositories queries

This commit is contained in:
Nikolaos Laskaris 2017-11-22 11:50:47 +02:00
parent e0a3f88279
commit 2348199dce
10 changed files with 153 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dmp-backend/src/main/java/proxy/config/ExternalUrls.java
View File

@ -19,6 +19,9 @@ public class ExternalUrls implements Serializable {
private static final long serialVersionUID = -5076364662014107275L;
Long maxresults;
ProjectUrls projects;
RegistryUrls registries;
RepositoryUrls repositories;
@ -85,6 +88,15 @@ public class ExternalUrls implements Serializable {
public void setProjects(ProjectUrls projects) {
this.projects = projects;
}
public Long getMaxresults() {
return maxresults;
}
@XmlElement(name = "maxresults")
public void setMaxresults(Long maxresults) {
this.maxresults = maxresults;
}

13
dmp-backend/src/main/java/proxy/config/exceptions/HugeResultSet.java Normal file
View File

@ -0,0 +1,13 @@
package proxy.config.exceptions;
public class HugeResultSet extends Exception {
private static final long serialVersionUID = -6961447213733280563L;
public HugeResultSet(String message) {
super(message);
}
}

24
dmp-backend/src/main/java/proxy/fetching/RemoteFetcher.java
View File

@ -27,6 +27,7 @@ import com.jayway.jsonpath.JsonPath;
import proxy.config.ConfigLoader;
import proxy.config.UrlConfig;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
@Service
@ -35,6 +36,8 @@ public class RemoteFetcher {
@Autowired private ConfigLoader configLoader;
// private static int MAX_RESULTS = 30;
// public static void main(String [] args) throws Exception {
//
@ -50,38 +53,37 @@ public class RemoteFetcher {
// }
@Cacheable("repositories")
public List<Map<String, String>> getRepositories(String query) throws NoURLFound {
System.out.println("NOT CACHED");
public List<Map<String, String>> getRepositories(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls();
return getAll(urlConfigs, query);
}
@Cacheable("projects")
public List<Map<String, String>> getProjects(String query) throws NoURLFound {
public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
return getAll(urlConfigs, query);
}
@Cacheable("organisations")
public List<Map<String, String>> getOrganisations(String query) throws NoURLFound {
public List<Map<String, String>> getOrganisations(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls();
return getAll(urlConfigs, query);
}
@Cacheable("registries")
public List<Map<String, String>> getRegistries(String query) throws NoURLFound {
public List<Map<String, String>> getRegistries(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls();
return getAll(urlConfigs, query);
}
@Cacheable("services")
public List<Map<String, String>> getServices(String query) throws NoURLFound {
public List<Map<String, String>> getServices(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getServices().getUrls();
return getAll(urlConfigs, query);
}
@Cacheable("researchers")
public List<Map<String, String>> getResearchers(String query) throws NoURLFound {
public List<Map<String, String>> getResearchers(String query) throws NoURLFound, HugeResultSet {
List<UrlConfig> urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls();
return getAll(urlConfigs, query);
}
@ -89,7 +91,7 @@ public class RemoteFetcher {
private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, String query) throws NoURLFound{
private List<Map<String, String>> getAll(List<UrlConfig> urlConfigs, String query) throws NoURLFound, HugeResultSet{
if(urlConfigs == null || urlConfigs.isEmpty())
throw new NoURLFound("No Repository urls found in configuration");
@ -107,7 +109,7 @@ public class RemoteFetcher {
private List<Map<String, String>> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) {
private List<Map<String, String>> getAllResultsFromUrl(String path, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet {
Set<Integer> pages = new HashSet<Integer>();
final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : "";
@ -119,6 +121,10 @@ public class RemoteFetcher {
for(int i = 2; i <= results.getPagination().get("pages") ; i++)
pages.add(i);
Long maxResults = configLoader.getExternalUrls().getMaxresults();
if( (maxResults > 0) && (results.getPagination().get("count") > maxResults) )
throw new HugeResultSet("The submitted search query "+query+" is about to return "+results.getPagination().get("count") +" results... Please submit a more detailed search query");
//remaining calls (if pages array has elements)
Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(path + "?page="+page + searchQuery, jsonDataPath, jsonPaginationPath))

14
dmp-backend/src/main/java/rest/entities/DataRepositories.java
View File

@ -2,6 +2,7 @@ package rest.entities;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@ -51,6 +52,8 @@ import entities.Researcher;
import entities.Service;
import helpers.SerializerProvider;
import helpers.Transformers;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
import proxy.fetching.RemoteFetcher;
import responses.RestResponse;
@ -76,16 +79,17 @@ public class DataRepositories {
@RequestMapping(method = RequestMethod.GET, value = { "/externaldatarepo" }, produces="application/json")
public @ResponseBody ResponseEntity<Object> listExternalDataRepositories(@RequestParam("query") String query ){
public @ResponseBody ResponseEntity<Object> listExternalDataRepositories(@RequestParam(value="query", required=false) String query ){
try {
List<Map<String,String>> remoteRepos = remoteFetcher.getRepositories(query);
return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos));
}
catch(Exception ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Serialization issue: "+ex.getMessage());
catch(NoURLFound ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("{\"reason\":\""+ex.getMessage()+"\"}");
}
catch(HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("{\"reason\":\""+ex.getMessage()+"\"}"); //the ex.getMessage has the appropriate text description
}
}

21
dmp-backend/src/main/java/rest/entities/Organisations.java
View File

@ -1,6 +1,7 @@
package rest.entities;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@ -49,6 +50,9 @@ import entities.Researcher;
import entities.Service;
import helpers.SerializerProvider;
import helpers.Transformers;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
import proxy.fetching.RemoteFetcher;
import responses.RestResponse;
@ -70,6 +74,23 @@ public class Organisations {
@Autowired private ServiceDao serviceDao;
@Autowired private RemoteFetcher remoteFetcher;
@RequestMapping(method = RequestMethod.GET, value = { "/externalorganisations" }, produces="application/json")
public @ResponseBody ResponseEntity<Object> listExternalOrganisations(@RequestParam(value="query", required=false) String query ){
try {
List<Map<String,String>> remoteRepos = remoteFetcher.getOrganisations(query);
return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos));
}
catch(NoURLFound ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("{\"reason\":\""+ex.getMessage()+"\"}");
}
catch(HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("{\"reason\":\""+ex.getMessage()+"\"}"); //the ex.getMessage has the appropriate text description
}
}
// MANAGE ORGANISATIONS(S)

20
dmp-backend/src/main/java/rest/entities/Projects.java
View File

@ -61,6 +61,9 @@ import entities.UserInfo;
import entities.responses.IDLabelPair;
import helpers.SerializerProvider;
import helpers.Transformers;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
import proxy.fetching.RemoteFetcher;
import responses.RestResponse;
@ -82,6 +85,23 @@ public class Projects {
@Autowired private ServiceDao serviceDao;
@Autowired private UserInfoDao userInfoDao;
@Autowired private RemoteFetcher remoteFetcher;
@RequestMapping(method = RequestMethod.GET, value = { "/externalprojects" }, produces="application/json")
public @ResponseBody ResponseEntity<Object> listExternalProjects(@RequestParam(value="query", required=false) String query ){
try {
List<Map<String,String>> remoteRepos = remoteFetcher.getProjects(query);
return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos));
}
catch(NoURLFound ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("{\"reason\":\""+ex.getMessage()+"\"}");
}
catch(HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("{\"reason\":\""+ex.getMessage()+"\"}"); //the ex.getMessage has the appropriate text description
}
}
// MANAGE PROJECT(S)

21
dmp-backend/src/main/java/rest/entities/Registries.java
View File

@ -1,6 +1,7 @@
package rest.entities;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@ -49,6 +50,9 @@ import entities.Service;
import entities.responses.IDLabelPair;
import helpers.SerializerProvider;
import helpers.Transformers;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
import proxy.fetching.RemoteFetcher;
import responses.RestResponse;
@ -69,6 +73,23 @@ public class Registries {
@Autowired private ResearcherDao researcherDao;
@Autowired private ServiceDao serviceDao;
@Autowired private RemoteFetcher remoteFetcher;
@RequestMapping(method = RequestMethod.GET, value = { "/externalregistries" }, produces="application/json")
public @ResponseBody ResponseEntity<Object> listExternalRegistries(@RequestParam(value="query", required=false) String query ){
try {
List<Map<String,String>> remoteRepos = remoteFetcher.getRegistries(query);
return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos));
}
catch(NoURLFound ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("{\"reason\":\""+ex.getMessage()+"\"}");
}
catch(HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("{\"reason\":\""+ex.getMessage()+"\"}"); //the ex.getMessage has the appropriate text description
}
}
// MANAGE REGISTRY(IES)

21
dmp-backend/src/main/java/rest/entities/Researchers.java
View File

@ -1,6 +1,7 @@
package rest.entities;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@ -49,6 +50,9 @@ import entities.Researcher;
import entities.Service;
import helpers.SerializerProvider;
import helpers.Transformers;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
import proxy.fetching.RemoteFetcher;
import responses.RestResponse;
@ -69,6 +73,23 @@ public class Researchers {
@Autowired private ResearcherDao researcherDao;
@Autowired private ServiceDao serviceDao;
@Autowired private RemoteFetcher remoteFetcher;
@RequestMapping(method = RequestMethod.GET, value = { "/externalresearchers" }, produces="application/json")
public @ResponseBody ResponseEntity<Object> listExternalResearchers(@RequestParam(value="query", required=false) String query ){
try {
List<Map<String,String>> remoteRepos = remoteFetcher.getResearchers(query);
return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos));
}
catch(NoURLFound ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("{\"reason\":\""+ex.getMessage()+"\"}");
}
catch(HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("{\"reason\":\""+ex.getMessage()+"\"}"); //the ex.getMessage has the appropriate text description
}
}
// MANAGE RESEARCHER(S)

20
dmp-backend/src/main/java/rest/entities/Services.java
View File

@ -2,6 +2,7 @@ package rest.entities;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@ -50,6 +51,9 @@ import entities.Researcher;
import entities.Service;
import helpers.SerializerProvider;
import helpers.Transformers;
import proxy.config.exceptions.HugeResultSet;
import proxy.config.exceptions.NoURLFound;
import proxy.fetching.RemoteFetcher;
import responses.RestResponse;
@ -70,6 +74,22 @@ public class Services {
@Autowired private ResearcherDao researcherDao;
@Autowired private ServiceDao serviceDao;
@Autowired private RemoteFetcher remoteFetcher;
@RequestMapping(method = RequestMethod.GET, value = { "/externalservices" }, produces="application/json")
public @ResponseBody ResponseEntity<Object> listExternalServices(@RequestParam(value="query", required=false) String query ){
try {
List<Map<String,String>> remoteRepos = remoteFetcher.getServices(query);
return ResponseEntity.status(HttpStatus.OK).body(SerializerProvider.toJson(remoteRepos));
}
catch(NoURLFound ex) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("{\"reason\":\""+ex.getMessage()+"\"}");
}
catch(HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body("{\"reason\":\""+ex.getMessage()+"\"}"); //the ex.getMessage has the appropriate text description
}
}
// MANAGE SERVICE(S)

1
dmp-backend/src/main/resources/ExternalUrls.xml
View File

@ -2,6 +2,7 @@
<externalUrls>
<maxresults>1000</maxresults> <!-- if you want it disabled, please enter a negative number -->
<registries>
<urls>