From 39acf89bb4d3a67f6a3d1a0cfd5685ab78d50d34 Mon Sep 17 00:00:00 2001 From: annabakouli Date: Wed, 17 Jan 2018 17:06:35 +0200 Subject: [PATCH] no message --- .../main/java/eu/eudat/controllers/Admin.java | 1 - .../eudat/controllers/DataRepositories.java | 9 +- .../controllers/DatasetWizardController.java | 2 +- .../eudat/controllers/ExternalDatasets.java | 6 +- .../java/eu/eudat/controllers/Registries.java | 3 + .../eu/eudat/dao/entities/DatasetDaoImpl.java | 2 +- .../entities/DatasetExternalDataset.java | 66 ++++ .../managers/ExternalDatasetManager.java | 4 +- .../datasetwizard/DatasetWizardModel.java | 26 +- .../eudat/proxy/fetching/RemoteFetcher.java | 340 +++++++++--------- .../dataset-wizard.component.html | 4 +- .../dataset-wizard.component.ts | 31 +- .../external-sources.service.ts | 1 + 13 files changed, 279 insertions(+), 216 deletions(-) create mode 100644 dmp-backend/src/main/java/eu/eudat/entities/DatasetExternalDataset.java diff --git a/dmp-backend/src/main/java/eu/eudat/controllers/Admin.java b/dmp-backend/src/main/java/eu/eudat/controllers/Admin.java index 9e5199437..2b5468def 100644 --- a/dmp-backend/src/main/java/eu/eudat/controllers/Admin.java +++ b/dmp-backend/src/main/java/eu/eudat/controllers/Admin.java @@ -38,7 +38,6 @@ public class Admin extends BaseController{ try{ eu.eudat.entities.DatasetProfile modelDefinition = AdminManager.generateViewStyleDefinition(profile); this.getApiContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(modelDefinition); - return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId()); }catch(Exception ex){ ex.printStackTrace(); diff --git a/dmp-backend/src/main/java/eu/eudat/controllers/DataRepositories.java b/dmp-backend/src/main/java/eu/eudat/controllers/DataRepositories.java index dacfad99d..822af96f2 100644 --- a/dmp-backend/src/main/java/eu/eudat/controllers/DataRepositories.java +++ b/dmp-backend/src/main/java/eu/eudat/controllers/DataRepositories.java @@ -7,6 +7,7 @@ import java.util.UUID; import javax.transaction.Transactional; import eu.eudat.entities.DataRepository; +import eu.eudat.models.helpers.responses.ResponseItem; import eu.eudat.services.ApiContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -47,16 +48,16 @@ public class DataRepositories extends BaseController{ } @RequestMapping(method = RequestMethod.GET, value = { "/external/datarepos" }, produces="application/json") - public @ResponseBody ResponseEntity>> listExternalDataRepositories(@RequestParam(value="query", required=false) String query ){ + public @ResponseBody ResponseItem>> listExternalDataRepositories(@RequestParam(value="query", required=false) String query ){ try { List> remoteRepos = this.getApiContext().getRemoteFetcher().getRepositories(query); - return ResponseEntity.status(HttpStatus.OK).body(remoteRepos); + return new ResponseItem>>().status(HttpStatus.OK).payload(remoteRepos); } catch(NoURLFound ex) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(null); + return new ResponseItem>>().status(HttpStatus.INTERNAL_SERVER_ERROR).payload(null); } catch(HugeResultSet ex) { - return ResponseEntity.status(HttpStatus.UNPROCESSABLE_ENTITY).body(null); + return new ResponseItem>>().status(HttpStatus.UNPROCESSABLE_ENTITY).payload(null); } } diff --git a/dmp-backend/src/main/java/eu/eudat/controllers/DatasetWizardController.java b/dmp-backend/src/main/java/eu/eudat/controllers/DatasetWizardController.java index 6be212d14..648cac2e3 100644 --- a/dmp-backend/src/main/java/eu/eudat/controllers/DatasetWizardController.java +++ b/dmp-backend/src/main/java/eu/eudat/controllers/DatasetWizardController.java @@ -77,7 +77,7 @@ public class DatasetWizardController extends BaseController{ } @Transactional - @RequestMapping(method = RequestMethod.POST, value = { "/createOrUpdate" }, produces="application/json") + @RequestMapping(method = RequestMethod.POST, value = { "/createOrUpdate" }, consumes = "application/json",produces="application/json") public @ResponseBody ResponseItem createOrUpdate(@RequestBody DatasetWizardModel profile,Principal principal) { try { eu.eudat.entities.Dataset dataset= DatasetManager.createOrUpdate(this.getApiContext(),profile); diff --git a/dmp-backend/src/main/java/eu/eudat/controllers/ExternalDatasets.java b/dmp-backend/src/main/java/eu/eudat/controllers/ExternalDatasets.java index 1d05a0076..ee9934d75 100644 --- a/dmp-backend/src/main/java/eu/eudat/controllers/ExternalDatasets.java +++ b/dmp-backend/src/main/java/eu/eudat/controllers/ExternalDatasets.java @@ -45,10 +45,10 @@ public class ExternalDatasets extends BaseController{ } } - @RequestMapping(method = RequestMethod.POST, value = {"/external/datasets"}, consumes = "application/json", produces = "application/json") - public @ResponseBody ResponseItem> getWithExternal(@RequestBody ExternalDatasetCriteria externalDatasetCriteria, Principal principal) { + @RequestMapping(method = RequestMethod.GET, value = {"/external/datasets"}, produces = "application/json") + public @ResponseBody ResponseItem> getWithExternal(@RequestParam(value = "query", required = false) String query, Principal principal) { try { - List dataTable = new ExternalDatasetManager().getWithExternal(this.getApiContext().getDatabaseRepository().getExternalDatasetDao(), externalDatasetCriteria, this.getApiContext().getRemoteFetcher()); + List dataTable = new ExternalDatasetManager().getWithExternal(this.getApiContext().getDatabaseRepository().getExternalDatasetDao(), query, this.getApiContext().getRemoteFetcher()); return new ResponseItem>().payload(dataTable).status(HttpStatus.OK); } catch (Exception ex) { ex.printStackTrace(); diff --git a/dmp-backend/src/main/java/eu/eudat/controllers/Registries.java b/dmp-backend/src/main/java/eu/eudat/controllers/Registries.java index 04c34c7ed..d7b97918e 100644 --- a/dmp-backend/src/main/java/eu/eudat/controllers/Registries.java +++ b/dmp-backend/src/main/java/eu/eudat/controllers/Registries.java @@ -58,11 +58,14 @@ public class Registries extends BaseController{ return new ResponseItem().payload(registriesExternalSourcesModel).status(HttpStatus.OK); } catch(NoURLFound ex) { + ex.printStackTrace(); return new ResponseItem().status(HttpStatus.BAD_REQUEST).message("External Url Not Found"); } catch(HugeResultSet ex) { + ex.printStackTrace(); return new ResponseItem().status(HttpStatus.BAD_REQUEST).message("Huge Result Set"); }catch (Exception ex){ + ex.printStackTrace(); return new ResponseItem().status(HttpStatus.BAD_REQUEST).message(ex.getMessage()); } } diff --git a/dmp-backend/src/main/java/eu/eudat/dao/entities/DatasetDaoImpl.java b/dmp-backend/src/main/java/eu/eudat/dao/entities/DatasetDaoImpl.java index 386ad835c..11c0fce15 100644 --- a/dmp-backend/src/main/java/eu/eudat/dao/entities/DatasetDaoImpl.java +++ b/dmp-backend/src/main/java/eu/eudat/dao/entities/DatasetDaoImpl.java @@ -37,7 +37,7 @@ public class DatasetDaoImpl implements DatasetDao { if (criteria.getStatus() != null)query.where((builder, root) -> builder.equal(root.get("status"), criteria.getStatus())); if (criteria.getPeriodEnd() != null)query.where((builder, root) -> builder.lessThan(root.get("created"), criteria.getPeriodEnd())); if (criteria.getPeriodStart() != null) query.where((builder, root) -> builder.greaterThan(root.get("created"), criteria.getPeriodStart())); - if (criteria.getDmpIds() != null && !criteria.getDmpIds().isEmpty()) query.where((builder, root) -> root.get("dmp.id").in( criteria.getDmpIds())); + if (criteria.getDmpIds() != null && !criteria.getDmpIds().isEmpty()) query.where((builder, root) -> root.get("dmp").get("id").in( criteria.getDmpIds())); return query; } diff --git a/dmp-backend/src/main/java/eu/eudat/entities/DatasetExternalDataset.java b/dmp-backend/src/main/java/eu/eudat/entities/DatasetExternalDataset.java new file mode 100644 index 000000000..ab1a94e15 --- /dev/null +++ b/dmp-backend/src/main/java/eu/eudat/entities/DatasetExternalDataset.java @@ -0,0 +1,66 @@ +package eu.eudat.entities; + +import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Type; + +import javax.persistence.*; +import java.util.UUID; + +/** + * Created by ikalyvas on 1/17/2018. + */ + +@Entity +@Table(name="\"DatasetExternalDataset\"") +public class DatasetExternalDataset { + + @Id + @GeneratedValue + @GenericGenerator(name = "uuid2", strategy = "uuid2") + @Column(name = "\"Id\"", updatable = false, nullable = false, columnDefinition = "BINARY(16)") + private UUID id; + + + @Type(type="org.hibernate.type.PostgresUUIDType") //DEPWARN dependency to Hibernate and PostgreSQL + @Column(name = "\"Dataset\"", nullable = false) + private UUID dataset; + + @Type(type="org.hibernate.type.PostgresUUIDType") //DEPWARN dependency to Hibernate and PostgreSQL + @Column(name = "\"ExternalDataset\"", nullable = false) + private UUID externalDataset; + + @Column(name = "\"Role\"") + private Integer role; + + public UUID getId() { + return id; + } + + public void setId(UUID id) { + this.id = id; + } + + public UUID getDataset() { + return dataset; + } + + public void setDataset(UUID dataset) { + this.dataset = dataset; + } + + public UUID getExternalDataset() { + return externalDataset; + } + + public void setExternalDataset(UUID externalDataset) { + this.externalDataset = externalDataset; + } + + public Integer getRole() { + return role; + } + + public void setRole(Integer role) { + this.role = role; + } +} diff --git a/dmp-backend/src/main/java/eu/eudat/managers/ExternalDatasetManager.java b/dmp-backend/src/main/java/eu/eudat/managers/ExternalDatasetManager.java index 4f06600d3..e2b83f3df 100644 --- a/dmp-backend/src/main/java/eu/eudat/managers/ExternalDatasetManager.java +++ b/dmp-backend/src/main/java/eu/eudat/managers/ExternalDatasetManager.java @@ -35,7 +35,9 @@ public class ExternalDatasetManager { return externalDatasetDataTableData; } - public List getWithExternal(ExternalDatasetDao externalDatasetDao , ExternalDatasetCriteria criteria, RemoteFetcher remoteFetcher) throws HugeResultSet, NoURLFound, InstantiationException, IllegalAccessException { + public List getWithExternal(ExternalDatasetDao externalDatasetDao , String query, RemoteFetcher remoteFetcher) throws HugeResultSet, NoURLFound, InstantiationException, IllegalAccessException { + ExternalDatasetCriteria criteria = new ExternalDatasetCriteria(); + criteria.setLike(query); QueryableList items = externalDatasetDao.getWithCriteria(criteria); List externalDatasets = new DomainModelConverter().fromDataModel(items.toList(), ExternalDatasetListingModel.class); return externalDatasets; diff --git a/dmp-backend/src/main/java/eu/eudat/models/datasetwizard/DatasetWizardModel.java b/dmp-backend/src/main/java/eu/eudat/models/datasetwizard/DatasetWizardModel.java index 91ac87ad5..e83087585 100644 --- a/dmp-backend/src/main/java/eu/eudat/models/datasetwizard/DatasetWizardModel.java +++ b/dmp-backend/src/main/java/eu/eudat/models/datasetwizard/DatasetWizardModel.java @@ -2,12 +2,15 @@ package eu.eudat.models.datasetwizard; import eu.eudat.entities.DMP; import eu.eudat.entities.Dataset; +import eu.eudat.entities.ExternalDataset; import eu.eudat.models.DataModel; import eu.eudat.models.dataset.DataRepository; import eu.eudat.models.dataset.Registry; import eu.eudat.models.dataset.Service; import eu.eudat.models.datasetprofile.DatasetProfileListingModel; import eu.eudat.models.dmp.DataManagementPlan; +import eu.eudat.models.externaldataset.ExternalDatasetListingModel; +import eu.eudat.models.listingmodels.DatasetListingModel; import eu.eudat.models.user.composite.DatasetProfile; import eu.eudat.utilities.builders.DomainModelConverter; @@ -32,6 +35,7 @@ public class DatasetWizardModel implements DataModel { private List registries; private List services; private List dataRepositories; + private List externalDatasets; private DatasetProfileListingModel profile; public UUID getId() { @@ -146,6 +150,14 @@ public class DatasetWizardModel implements DataModel { this.profile = profile; } + public List getExternalDatasets() { + return externalDatasets; + } + + public void setExternalDatasets(List externalDatasets) { + this.externalDatasets = externalDatasets; + } + @Override public void fromDataModel(Dataset entity) throws InstantiationException, IllegalAccessException { this.id = entity.getId(); @@ -160,6 +172,7 @@ public class DatasetWizardModel implements DataModel { this.services = new DomainModelConverter().fromDataModel(entity.getServices().stream().collect(Collectors.toList()), Service.class); this.created = entity.getCreated(); this.dmp = new DomainModelConverter().fromDataModel(Arrays.asList(entity.getDmp()), DataManagementPlan.class).get(0); + this.externalDatasets = new DomainModelConverter().fromDataModel(entity.getExternalDatasets().stream().collect(Collectors.toList()), ExternalDatasetListingModel.class); } @Override @@ -176,26 +189,33 @@ public class DatasetWizardModel implements DataModel { entity.setCreated(this.created != null ? this.created : new Date()); entity.setModified(new Date()); entity.setProfile(profile.toDataModel()); ///TODO - if (!this.registries.isEmpty()) { + if (this.registries != null && !this.registries.isEmpty()) { entity.setRegistries(new HashSet()); for (Registry registry : this.registries) { entity.getRegistries().add(registry.toDataModel()); } } - if (!this.dataRepositories.isEmpty()) { + if (this.dataRepositories != null && !this.dataRepositories.isEmpty()) { entity.setDataRepositories(new HashSet()); for (DataRepository dataRepository : this.dataRepositories) { entity.getDataRepositories().add(dataRepository.toDataModel()); } } - if (!this.services.isEmpty()) { + if (this.services != null && !this.services.isEmpty()) { entity.setServices(new HashSet()); for (Service service : this.services) { entity.getServices().add(service.toDataModel()); } } + + if (this.externalDatasets != null && !this.externalDatasets.isEmpty()) { + entity.setExternalDatasets(new HashSet()); + for (ExternalDatasetListingModel externalDataset : this.externalDatasets) { + entity.getExternalDatasets().add(externalDataset.toDataModel()); + } + } return entity; } } diff --git a/dmp-backend/src/main/java/eu/eudat/proxy/fetching/RemoteFetcher.java b/dmp-backend/src/main/java/eu/eudat/proxy/fetching/RemoteFetcher.java index 5b0e64d6d..6b2c47682 100644 --- a/dmp-backend/src/main/java/eu/eudat/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/src/main/java/eu/eudat/proxy/fetching/RemoteFetcher.java @@ -4,14 +4,7 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -30,175 +23,172 @@ import eu.eudat.proxy.config.exceptions.NoURLFound; @Service public class RemoteFetcher { - - - @Autowired private ConfigLoader configLoader; - + + + @Autowired + private ConfigLoader configLoader; + // private static int MAX_RESULTS = 30; - - - @Cacheable("repositories") - public List> getRepositories(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - - @Cacheable("projects") - public List> getProjects(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - - @Cacheable("organisations") - public List> getOrganisations(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - - @Cacheable("registries") - public List> getRegistries(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - - @Cacheable("services") - public List> getServices(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getServices().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - - @Cacheable("researchers") - public List> getResearchers(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - @Cacheable("datasets") - public List> getDatasets(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = configLoader.getExternalUrls().getDatasets().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); - } - - - - - private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet{ - - if(urlConfigs == null || urlConfigs.isEmpty()) - throw new NoURLFound("No Repository urls found in configuration"); - - Collections.sort(urlConfigs, (config1, config2) -> config1.getOrdinal().compareTo(config2.getOrdinal())); - return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), fetchStrategy, urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query); - - } - - - - - - private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet { - Set pages = new HashSet(); - - final String searchQuery = (query!=null) && !query.isEmpty() ? "&search="+query : ""; - - //first call - Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath); - - //if fetch strategy is to get only first page, then return that - if(fetchStrategy == FetchStrategy.FIRST) - return results.getResults(); - - if(results.getPagination()!= null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set - for(int i = 2; i <= results.getPagination().get("pages") ; i++) - pages.add(i); - - Long maxResults = configLoader.getExternalUrls().getMaxresults(); - if( (maxResults > 0) && (results.getPagination().get("count") > maxResults) ) - throw new HugeResultSet("The submitted search query "+query+" is about to return "+results.getPagination().get("count") +" results... Please submit a more detailed search query"); - - //remaining calls (if pages array has elements) - Optional optionalResults = pages.parallelStream() - .map(page -> getResultsFromUrl(path + "?page="+page + searchQuery, jsonDataPath, jsonPaginationPath)) - .reduce((result1, result2) -> { - result1.getResults().addAll(result2.getResults()); - return result1; - }); - Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results(); - - remainingResults.getResults().addAll(results.getResults()); - - return remainingResults.getResults(); - } - - - private Results getResultsFromUrl(String urlString, String jsonDataPath, String jsonPaginationPath) { - - try { - - URL url = new URL(urlString); - - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod("GET"); - con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8"); - - int responseCode = con.getResponseCode(); - if (responseCode == HttpURLConnection.HTTP_OK) { // success - //do here all the parsing - DocumentContext jsonContext = JsonPath.parse(con.getInputStream()); - Results results = new Results(jsonContext.read(jsonDataPath), jsonContext.read(jsonPaginationPath)); - return results; - } - } - catch(MalformedURLException e1) {} //maybe print smth... - catch(IOException e2) {} //maybe print smth... - finally {} - - return null; - - } - - - - - class Results { - - List> results; - Map pagination; - - public Results() { - this.results = new ArrayList>(); - this.pagination = new HashMap(); - } - - public Results(List> results, Map pagination) { - this.results = results; - this.pagination = pagination; - } - - public List> getResults() { - return results; - } - public void setResults(List> results) { - this.results = results; - } - public Map getPagination() { - return pagination; - } - public void setPagination(Map pagination) { - this.pagination = pagination; - } - - - - - } + @Cacheable("repositories") + public List> getRepositories(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getRepositories().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + @Cacheable("projects") + public List> getProjects(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + @Cacheable("organisations") + public List> getOrganisations(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getOrganisations().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + @Cacheable("registries") + public List> getRegistries(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getRegistries().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + @Cacheable("services") + public List> getServices(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getServices().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + @Cacheable("researchers") + public List> getResearchers(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getResearchers().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + @Cacheable("datasets") + public List> getDatasets(String query) throws NoURLFound, HugeResultSet { + List urlConfigs = configLoader.getExternalUrls().getDatasets().getUrls(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode(); + return getAll(urlConfigs, fetchStrategy, query); + } + + + private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet { + + if (urlConfigs == null || urlConfigs.isEmpty()) + throw new NoURLFound("No Repository urls found in configuration"); + + Collections.sort(urlConfigs, (config1, config2) -> config1.getOrdinal().compareTo(config2.getOrdinal())); + + return getAllResultsFromUrl(urlConfigs.get(0).getUrl(), fetchStrategy, urlConfigs.get(0).getDataPath(), urlConfigs.get(0).getPaginationPath(), query); + + } + + + private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final String jsonDataPath, final String jsonPaginationPath, String query) throws HugeResultSet { + Set pages = new HashSet(); + + final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : ""; + + //first call + Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath); + + //if fetch strategy is to get only first page, then return that + if (fetchStrategy == FetchStrategy.FIRST) + return results == null ? new LinkedList<>() : results.getResults(); + + if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set + for (int i = 2; i <= results.getPagination().get("pages"); i++) + pages.add(i); + + Long maxResults = configLoader.getExternalUrls().getMaxresults(); + if ((maxResults > 0) && (results.getPagination().get("count") > maxResults)) + throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); + + //remaining calls (if pages array has elements) + Optional optionalResults = pages.parallelStream() + .map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath)) + .reduce((result1, result2) -> { + result1.getResults().addAll(result2.getResults()); + return result1; + }); + Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results(); + + remainingResults.getResults().addAll(results.getResults()); + + return remainingResults.getResults(); + } + + + private Results getResultsFromUrl(String urlString, String jsonDataPath, String jsonPaginationPath) { + + try { + + URL url = new URL(urlString); + + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("GET"); + con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8"); + + int responseCode = con.getResponseCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { // success + //do here all the parsing + DocumentContext jsonContext = JsonPath.parse(con.getInputStream()); + Results results = new Results(jsonContext.read(jsonDataPath), jsonContext.read(jsonPaginationPath)); + return results; + } + } catch (MalformedURLException e1) { + } //maybe print smth... + catch (IOException e2) { + } //maybe print smth... + finally { + } + + return null; + + } + + + class Results { + + List> results; + Map pagination; + + public Results() { + this.results = new ArrayList>(); + this.pagination = new HashMap(); + } + + public Results(List> results, Map pagination) { + this.results = results; + this.pagination = pagination; + } + + public List> getResults() { + return results; + } + + public void setResults(List> results) { + this.results = results; + } + + public Map getPagination() { + return pagination; + } + + public void setPagination(Map pagination) { + this.pagination = pagination; + } + + + } } diff --git a/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.html b/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.html index 38a27ab5d..0da875d67 100644 --- a/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.html +++ b/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.html @@ -72,11 +72,11 @@ (inputChange)="filterExternalDatasets($event)" requireMatch>
{{chip.label.substring(0, 1).toUpperCase()}}
- {{chip.name}} + {{chip.label}}
- {{option.name}} + {{option.label}}
diff --git a/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.ts b/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.ts index 6f980c25d..b752beff7 100644 --- a/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.ts +++ b/dmp-frontend/src/app/dataset-wizard/dataset-wizard.component.ts @@ -17,9 +17,9 @@ import { Observable } from "rxjs/Observable"; import { RequestItem } from "../models/criteria/RequestItem"; import { MatPaginator, MatSort, MatSnackBar } from "@angular/material"; import { SnackBarNotificationComponent } from "../shared/components/notificaiton/snack-bar-notification.component"; -import { ExternalSourcesItemModel } from '@app/models/external-sources/ExternalSourcesItemModel'; -import { ExternalSourcesService } from '@app/services/external-sources/external-sources.service'; -import { DatasetProfileModel } from '@app/models/datasetprofile/DatasetProfileModel'; +import { DatasetProfileModel } from '../models/datasetprofile/DatasetProfileModel'; +import { ExternalSourcesItemModel } from '../models/external-sources/ExternalSourcesItemModel'; +import { ExternalSourcesService } from '../services/external-sources/external-sources.service'; @Component({ selector: 'app-dataset-wizard-component', @@ -81,11 +81,6 @@ export class DatasetWizardComponent implements AfterViewInit { .subscribe(data => { this.datasetWizardModel = JsonSerializer.fromJSONObject(data, DatasetWizardModel); this.formGroup = this.datasetWizardModel.buildForm(); - //this.datasetProfileAutoCompleteConfiguration = new AutoCompleteConfiguration(this.datasetWizardService.getAvailableProfiles.bind(this.datasetWizardService), datasetProfileRequestItem); - // this.formGroup.get("dmp").valueChanges.subscribe(change => { - // this.formGroup.get('profile').setValue(null, { emitEvent: false }); - // //this.datasetProfileAutoCompleteConfiguration.requestItem.criteria.id = change.id - // }); }); } else if (dmpId != null) { this.isNew = true; @@ -93,24 +88,10 @@ export class DatasetWizardComponent implements AfterViewInit { .subscribe(data => { this.datasetWizardModel = new DatasetWizardModel(); setTimeout(() => { + this.datasetWizardModel.dmp = data; this.formGroup = this.datasetWizardModel.buildForm(); - this.datasetWizardModel.dmp = data; this.loadDatasetProfiles(); - //this.datasetProfileAutoCompleteConfiguration = new AutoCompleteConfiguration(this.datasetWizardService.getAvailableProfiles.bind(this.datasetWizardService), datasetProfileRequestItem); - // this.formGroup.get("dmp").valueChanges.subscribe(change => { - // this.formGroup.get('profile').setValue(null, { emitEvent: false }); - // //this.datasetProfileAutoCompleteConfiguration.requestItem.criteria.id = change.id - // }); }); - - // this.datasetWizardModel = JsonSerializer.fromJSONObject(data, DatasetWizardModel); - // this.formGroup = this.datasetWizardModel.buildForm(); - //this.datasetProfileAutoCompleteConfiguration = new AutoCompleteConfiguration(this.datasetWizardService.getAvailableProfiles.bind(this.datasetWizardService), datasetProfileRequestItem); - // this.formGroup.get("dmp").valueChanges.subscribe(change => { - // this.formGroup.get('profile').setValue(null, { emitEvent: false }); - - // //this.datasetProfileAutoCompleteConfiguration.requestItem.criteria.id = change.id - // }); }); } else { @@ -203,7 +184,7 @@ export class DatasetWizardComponent implements AfterViewInit { if (value) { this.filtereddataRepositoriesAsync = true; - this.externalSourcesService.searchDMPOrganizations(value).subscribe(items => { + this.externalSourcesService.searchDatasetRepository(value).subscribe(items => { this.filtereddataRepositories = items; this.filtereddataRepositoriesAsync = false; @@ -221,7 +202,7 @@ export class DatasetWizardComponent implements AfterViewInit { if (value) { this.filteredRegistriesAsync = true; - this.externalSourcesService.searchDMPResearchers(value).subscribe(items => { + this.externalSourcesService.searchDatasetRegistry(value).subscribe(items => { this.filteredRegistries = items; this.filteredRegistriesAsync = false; diff --git a/dmp-frontend/src/app/services/external-sources/external-sources.service.ts b/dmp-frontend/src/app/services/external-sources/external-sources.service.ts index 666f96c7a..6a9a37425 100644 --- a/dmp-frontend/src/app/services/external-sources/external-sources.service.ts +++ b/dmp-frontend/src/app/services/external-sources/external-sources.service.ts @@ -49,4 +49,5 @@ export class ExternalSourcesService { public searchDMPProfiles(like: string): Observable { return this.http.get(this.actionUrl + "datasetprofiles/get" + "?query=" + like, { headers: this.headers }); } + }