diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ExternalDatasetCriteria.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ExternalDatasetCriteria.java index f2e775366..a5a6275e8 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ExternalDatasetCriteria.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ExternalDatasetCriteria.java @@ -1,7 +1,15 @@ package eu.eudat.data.dao.criteria; - import eu.eudat.data.entities.ExternalDataset; +import java.util.UUID; public class ExternalDatasetCriteria extends Criteria { + private UUID creationUserId; + + public UUID getCreationUserId() { + return creationUserId; + } + public void setCreationUserId(UUID creationUserId) { + this.creationUserId = creationUserId; + } } diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ExternalDatasetDaoImpl.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ExternalDatasetDaoImpl.java index 9461faaca..163eed579 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ExternalDatasetDaoImpl.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ExternalDatasetDaoImpl.java @@ -28,6 +28,8 @@ public class ExternalDatasetDaoImpl extends DatabaseAccess impl query.where((builder, root) -> builder.or( builder.like(builder.upper(root.get("reference")), "%" + criteria.getLike().toUpperCase() + "%"), builder.equal(builder.upper(root.get("label")), criteria.getLike().toUpperCase()))); + if (criteria.getCreationUserId() != null) + query.where((builder, root) -> builder.equal(root.join("creationUser").get("id"), criteria.getCreationUserId())); return query; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java index 0e0bcc339..fc1dba6ae 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java +++ b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java @@ -36,6 +36,7 @@ public class ResponsesCache { caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); simpleCacheManager.setCaches(caches); System.out.println("OK"); return simpleCacheManager; diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalDatasets.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalDatasets.java index 040823492..532cacdac 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalDatasets.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalDatasets.java @@ -48,7 +48,7 @@ public class ExternalDatasets extends BaseController { ResponseEntity>> getWithExternal( @RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type, Principal principal ) throws NoURLFound, InstantiationException, HugeResultSet, IllegalAccessException { - List dataTable = externalDatasetManager.getWithExternal(query); + List dataTable = externalDatasetManager.getWithExternal(query, type, principal); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem>().payload(dataTable).status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index 653f3bc0d..5b04ef777 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -515,13 +515,15 @@ public class DatasetManager { if (dataset.getDatasetExternalDatasets() != null && !dataset.getDatasetExternalDatasets().isEmpty()) { for (eu.eudat.data.entities.DatasetExternalDataset datasetExternalDataset : dataset.getDatasetExternalDatasets()) { ExternalDatasetCriteria criteria = new ExternalDatasetCriteria(); - criteria.setLike(datasetExternalDataset.getExternalDataset().getLabel()); + criteria.setLike(datasetExternalDataset.getExternalDataset().getReference()); List entries = databaseRepository.getExternalDatasetDao().getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId()); datasetExternalDataset.setDataset(dataset); dataset.getDatasetExternalDatasets().add(datasetExternalDataset); } else { + datasetExternalDataset.getExternalDataset().setId(UUID.randomUUID()); + datasetExternalDataset.setDataset(dataset); ExternalDataset externalDataset = databaseRepository.getExternalDatasetDao().createOrUpdate(datasetExternalDataset.getExternalDataset()); datasetExternalDataset.setExternalDataset(externalDataset); dataset.getDatasetExternalDatasets().add(datasetExternalDataset); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ExternalDatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ExternalDatasetManager.java index 8c6417c20..f8f72ce31 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ExternalDatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ExternalDatasetManager.java @@ -1,12 +1,18 @@ package eu.eudat.logic.managers; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.logic.builders.model.criteria.ExternalDatasetCriteriaBuilder; import eu.eudat.logic.builders.model.models.DataTableDataBuilder; import eu.eudat.data.entities.ExternalDataset; import eu.eudat.data.dao.criteria.ExternalDatasetCriteria; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.services.operations.DatabaseRepository; +import eu.eudat.models.data.external.ExternalDatasetSourcesModel; +import eu.eudat.models.data.external.ExternalSourcesItemModel; import eu.eudat.models.data.externaldataset.ExternalDatasetListingModel; import eu.eudat.data.query.items.table.externaldataset.ExternalDatasetTableRequest; +import eu.eudat.models.data.externaldataset.ExternalDatasetModel; import eu.eudat.models.data.helpers.common.DataTableData; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; @@ -17,8 +23,11 @@ import eu.eudat.logic.services.ApiContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; @Component public class ExternalDatasetManager { @@ -40,10 +49,26 @@ public class ExternalDatasetManager { return apiContext.getOperationsContext().getBuilderFactory().getBuilder(DataTableDataBuilder.class).data(externalDatasetListingmodels).totalCount(items.count()).build(); } - public List getWithExternal(String query) throws HugeResultSet, NoURLFound, InstantiationException, IllegalAccessException { + public List getWithExternal(String query, String type, Principal principal) throws HugeResultSet, NoURLFound { + // Fetch the local saved external Datasets that belong to the user. ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build(); + criteria.setCreationUserId(principal.getId()); QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria); + + // Fetch external Datasets from external sources. + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); + List> remoteRepos = remoteFetcher.getDatasets(externalUrlCriteria, type); + + // Parse items from external sources to listing models. + ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + List externalDatasetModels = remoteRepos.stream() + .map(item -> mapper.convertValue(item, ExternalDatasetListingModel.class)) + .collect(Collectors.toCollection(LinkedList::new)); + + // Merge fetched and local. List externalDatasets = items.select(item -> new ExternalDatasetListingModel().fromDataModel(item)); + externalDatasets.addAll(externalDatasetModels); + return externalDatasets; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index b4422099a..efc0b1db4 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -108,7 +108,7 @@ public class RemoteFetcher { return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } - @Cacheable("datasets") + @Cacheable("externalDatasets") public List> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/DataRepository.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/DataRepository.java index f8b61726d..9c1a84201 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/DataRepository.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/DataRepository.java @@ -85,11 +85,17 @@ public class DataRepository implements DataModel { +public class ExternalDatasetSourcesModel extends ExternalListingItem { @Override - public ExternalDatasetModel fromExternalItem(List> values) { + public ExternalDatasetSourcesModel fromExternalItem(List> values) { for (Map item : values) { ExternalSourcesItemModel model = new ExternalSourcesItemModel(); model.setId(item.get("pid")); model.setUri(item.get("uri")); model.setName(item.get("name")); + model.setSource(item.get("source")); this.add(model); } return this; diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalSourcesItemModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalSourcesItemModel.java index 521563627..cc35ff289 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalSourcesItemModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalSourcesItemModel.java @@ -3,6 +3,7 @@ package eu.eudat.models.data.external; public class ExternalSourcesItemModel { private String id; + private String pid; private String name; private String description; private String uri; @@ -14,15 +15,20 @@ public class ExternalSourcesItemModel { public String getId() { return id; } - public void setId(String id) { this.id = id; } + public String getPid() { + return pid; + } + public void setPid(String pid) { + this.pid = pid; + } + public String getName() { return name; } - public void setName(String name) { this.name = name; } @@ -30,7 +36,6 @@ public class ExternalSourcesItemModel { public String getDescription() { return description; } - public void setDescription(String description) { this.description = description; } @@ -38,7 +43,6 @@ public class ExternalSourcesItemModel { public String getUri() { return uri; } - public void setUri(String uri) { this.uri = uri; } @@ -46,7 +50,6 @@ public class ExternalSourcesItemModel { public String getRemoteId() { return remoteId; } - public void setRemoteId(String remoteId) { this.remoteId = remoteId; } @@ -54,7 +57,6 @@ public class ExternalSourcesItemModel { public String getAbbreviation() { return abbreviation; } - public void setAbbreviation(String abbreviation) { this.abbreviation = abbreviation; } @@ -62,7 +64,6 @@ public class ExternalSourcesItemModel { public String getTag() { return tag; } - public void setTag(String tag) { this.tag = tag; } @@ -70,7 +71,6 @@ public class ExternalSourcesItemModel { public String getSource() { return source; } - public void setSource(String source) { this.source = source; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/externaldataset/ExternalDatasetListingModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/externaldataset/ExternalDatasetListingModel.java index 5b5472ced..8b258a292 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/externaldataset/ExternalDatasetListingModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/externaldataset/ExternalDatasetListingModel.java @@ -10,34 +10,35 @@ import java.util.UUID; public class ExternalDatasetListingModel implements DataModel { private UUID id; - private String label; + private String name; private String abbreviation; private String reference; private Date created; private Date modified; private String info; private ExternalDatasetType type; + private String pid; + private String uri; + private String tag; // Api fetching the data + private String source; // Actual harvested source public UUID getId() { return id; } - public void setId(UUID id) { this.id = id; } - public String getLabel() { - return label; + public String getName() { + return name; } - - public void setLabel(String label) { - this.label = label; + public void setName(String name) { + this.name = name; } public String getAbbreviation() { return abbreviation; } - public void setAbbreviation(String abbreviation) { this.abbreviation = abbreviation; } @@ -45,7 +46,6 @@ public class ExternalDatasetListingModel implements DataModel

- {{i+1}}) {{suggestion.get('label').value}} + {{i+1}}) {{suggestion.get('name').value}}

diff --git a/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/dataset-external-references-editor.component.ts b/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/dataset-external-references-editor.component.ts index 5faca89bc..6b3ba5f13 100644 --- a/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/dataset-external-references-editor.component.ts +++ b/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/dataset-external-references-editor.component.ts @@ -77,9 +77,9 @@ export class DatasetExternalReferencesEditorComponent extends BaseComponent impl filterFn: this.searchDatasetExternalDatasets.bind(this), removeAfterSelection: true, initialItems: (type) => this.searchDatasetExternalDatasets('', type),//.filter(resultItem => excludedItems.map(x => x.id).indexOf(resultItem.id) === -1), - displayFn: (item) => item ? item.label : null, - titleFn: (item) => item ? item.label : null, - subtitleFn: (item) => item.tag ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.tag : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE') + displayFn: (item) => item ? item.name : null, + titleFn: (item) => item ? item.name : null, + subtitleFn: (item) => item.source ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.source : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE') }; this.registriesAutoCompleteConfiguration = { @@ -97,7 +97,7 @@ export class DatasetExternalReferencesEditorComponent extends BaseComponent impl initialItems: (type) => this.searchDatasetExternalServices('', type), displayFn: (item) => item ? item.label : null, titleFn: (item) => item ? item.label : null, - subtitleFn: (item) => item.tag ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.tag : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE') + subtitleFn: (item) => item.source ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.source : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE') }; this.tagsAutoCompleteConfiguration = { @@ -115,7 +115,7 @@ export class DatasetExternalReferencesEditorComponent extends BaseComponent impl } externalDatasetsOnItemChange(event) { - const externalDatasetModel = new ExternalDatasetEditorModel(event.id, event.abbreviation, event.label, event.reference, event.source); + const externalDatasetModel = new ExternalDatasetEditorModel(event.id, event.abbreviation, event.name, event.pid ? event.pid : event.reference, event.source); (this.formGroup.get('externalDatasets')).push(externalDatasetModel.buildForm()); } diff --git a/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/editors/external-dataset/dataset-external-dataset-dialog-editor.component.html b/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/editors/external-dataset/dataset-external-dataset-dialog-editor.component.html index 6a0263865..530a33bd4 100644 --- a/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/editors/external-dataset/dataset-external-dataset-dialog-editor.component.html +++ b/dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/editors/external-dataset/dataset-external-dataset-dialog-editor.component.html @@ -9,8 +9,8 @@
- - {{'GENERAL.VALIDATION.REQUIRED' | translate}} + + {{'GENERAL.VALIDATION.REQUIRED' | translate}}