MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Adds remote fetching on "External Dataset" and fixes bug on "external references" not saving the correct "reference" property.

This commit is contained in:
gkolokythas 2019-11-19 11:01:02 +02:00
parent 1ef0782989
commit 2b62cdbe4e
18 changed files with 159 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/ExternalDatasetCriteria.java
View File

@ -1,7 +1,15 @@
package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.ExternalDataset;
import java.util.UUID;
public class ExternalDatasetCriteria extends Criteria<ExternalDataset> {
private UUID creationUserId;
public UUID getCreationUserId() {
return creationUserId;
}
public void setCreationUserId(UUID creationUserId) {
this.creationUserId = creationUserId;
}
}

2
dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/ExternalDatasetDaoImpl.java
View File

@ -28,6 +28,8 @@ public class ExternalDatasetDaoImpl extends DatabaseAccess<ExternalDataset> impl
query.where((builder, root) -> builder.or(
builder.like(builder.upper(root.get("reference")), "%" + criteria.getLike().toUpperCase() + "%"),
builder.equal(builder.upper(root.get("label")), criteria.getLike().toUpperCase())));
if (criteria.getCreationUserId() != null)
query.where((builder, root) -> builder.equal(root.join("creationUser").get("id"), criteria.getCreationUserId()));
return query;
}

1
dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java vendored
View File

@ -36,6 +36,7 @@ public class ResponsesCache {
caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
simpleCacheManager.setCaches(caches);
System.out.println("OK");
return simpleCacheManager;

2
dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalDatasets.java
View File

@ -48,7 +48,7 @@ public class ExternalDatasets extends BaseController {
ResponseEntity<ResponseItem<List<ExternalDatasetListingModel>>> getWithExternal(
@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type, Principal principal
) throws NoURLFound, InstantiationException, HugeResultSet, IllegalAccessException {
List<ExternalDatasetListingModel> dataTable = externalDatasetManager.getWithExternal(query);
List<ExternalDatasetListingModel> dataTable = externalDatasetManager.getWithExternal(query, type, principal);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<ExternalDatasetListingModel>>().payload(dataTable).status(ApiMessageCode.NO_MESSAGE));
}

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java
View File

@ -515,13 +515,15 @@ public class DatasetManager {
if (dataset.getDatasetExternalDatasets() != null && !dataset.getDatasetExternalDatasets().isEmpty()) {
for (eu.eudat.data.entities.DatasetExternalDataset datasetExternalDataset : dataset.getDatasetExternalDatasets()) {
ExternalDatasetCriteria criteria = new ExternalDatasetCriteria();
criteria.setLike(datasetExternalDataset.getExternalDataset().getLabel());
criteria.setLike(datasetExternalDataset.getExternalDataset().getReference());
List<eu.eudat.data.entities.ExternalDataset> entries = databaseRepository.getExternalDatasetDao().getWithCriteria(criteria).toList();
if (entries != null && !entries.isEmpty()) {
datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId());
datasetExternalDataset.setDataset(dataset);
dataset.getDatasetExternalDatasets().add(datasetExternalDataset);
} else {
datasetExternalDataset.getExternalDataset().setId(UUID.randomUUID());
datasetExternalDataset.setDataset(dataset);
ExternalDataset externalDataset = databaseRepository.getExternalDatasetDao().createOrUpdate(datasetExternalDataset.getExternalDataset());
datasetExternalDataset.setExternalDataset(externalDataset);
dataset.getDatasetExternalDatasets().add(datasetExternalDataset);

27
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ExternalDatasetManager.java
View File

@ -1,12 +1,18 @@
package eu.eudat.logic.managers;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.logic.builders.model.criteria.ExternalDatasetCriteriaBuilder;
import eu.eudat.logic.builders.model.models.DataTableDataBuilder;
import eu.eudat.data.entities.ExternalDataset;
import eu.eudat.data.dao.criteria.ExternalDatasetCriteria;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.models.data.external.ExternalDatasetSourcesModel;
import eu.eudat.models.data.external.ExternalSourcesItemModel;
import eu.eudat.models.data.externaldataset.ExternalDatasetListingModel;
import eu.eudat.data.query.items.table.externaldataset.ExternalDatasetTableRequest;
import eu.eudat.models.data.externaldataset.ExternalDatasetModel;
import eu.eudat.models.data.helpers.common.DataTableData;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
@ -17,8 +23,11 @@ import eu.eudat.logic.services.ApiContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@Component
public class ExternalDatasetManager {
@ -40,10 +49,26 @@ public class ExternalDatasetManager {
return apiContext.getOperationsContext().getBuilderFactory().getBuilder(DataTableDataBuilder.class).data(externalDatasetListingmodels).totalCount(items.count()).build();
}
public List<ExternalDatasetListingModel> getWithExternal(String query) throws HugeResultSet, NoURLFound, InstantiationException, IllegalAccessException {
public List<ExternalDatasetListingModel> getWithExternal(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
// Fetch the local saved external Datasets that belong to the user.
ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
criteria.setCreationUserId(principal.getId());
QueryableList<eu.eudat.data.entities.ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
// Fetch external Datasets from external sources.
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = remoteFetcher.getDatasets(externalUrlCriteria, type);
// Parse items from external sources to listing models.
ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
List<ExternalDatasetListingModel> externalDatasetModels = remoteRepos.stream()
.map(item -> mapper.convertValue(item, ExternalDatasetListingModel.class))
.collect(Collectors.toCollection(LinkedList::new));
// Merge fetched and local.
List<ExternalDatasetListingModel> externalDatasets = items.select(item -> new ExternalDatasetListingModel().fromDataModel(item));
externalDatasets.addAll(externalDatasetModels);
return externalDatasets;
}

2
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -108,7 +108,7 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("datasets")
@Cacheable("externalDatasets")
public List<Map<String, String>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())

16
dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/DataRepository.java
View File

@ -85,11 +85,17 @@ public class DataRepository implements DataModel<eu.eudat.data.entities.DataRepo
entity.setCreated(new Date());
entity.setModified(new Date());
entity.setStatus((short) 0);
if (this.source == null) this.source = "dmp";
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source + ":" + this.reference);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source + ":" + this.reference);
}
}
if (this.reference == null) {
entity.setReference(this.source + ":" + this.label);
}
return entity;
}

16
dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/Registry.java
View File

@ -89,11 +89,17 @@ public class Registry implements DataModel<eu.eudat.data.entities.Registry, Regi
entity.setId(this.id != null ? this.id : UUID.randomUUID());
entity.setLabel(this.label);
entity.setAbbreviation(this.abbreviation);
if (this.source == null) this.source = "dmp";
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source + ":" + this.reference);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source + ":" + this.reference);
}
}
if (this.reference == null) {
entity.setReference(this.source + ":" + this.label);
}
entity.setUri(this.uri);
entity.setModified(new Date());

17
dmp-backend/web/src/main/java/eu/eudat/models/data/dataset/Service.java
View File

@ -86,12 +86,17 @@ public class Service implements DataModel<eu.eudat.data.entities.Service, Servic
entity.setId(this.id != null ? this.id : UUID.randomUUID());
entity.setLabel(this.label);
entity.setAbbreviation(this.abbreviation);
if (this.source == null) this.source = "dmp";
if (this.reference == null) this.reference = entity.getId().toString();
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source + ":" + this.reference);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source + ":" + this.reference);
}
}
if (this.reference == null) {
entity.setReference(this.source + ":" + this.label);
}
entity.setUri(this.uri);
entity.setModified(new Date());

5
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalDatasetModel.java → dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalDatasetSourcesModel.java vendored
View File

@ -4,14 +4,15 @@ import java.util.List;
import java.util.Map;
public class ExternalDatasetModel extends ExternalListingItem<ExternalDatasetModel> {
public class ExternalDatasetSourcesModel extends ExternalListingItem<ExternalDatasetSourcesModel> {
@Override
public ExternalDatasetModel fromExternalItem(List<Map<String, String>> values) {
public ExternalDatasetSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setSource(item.get("source"));
this.add(model);
}
return this;

16
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalSourcesItemModel.java vendored
View File

@ -3,6 +3,7 @@ package eu.eudat.models.data.external;
public class ExternalSourcesItemModel {
private String id;
private String pid;
private String name;
private String description;
private String uri;
@ -14,15 +15,20 @@ public class ExternalSourcesItemModel {
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getPid() {
return pid;
}
public void setPid(String pid) {
this.pid = pid;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@ -30,7 +36,6 @@ public class ExternalSourcesItemModel {
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
@ -38,7 +43,6 @@ public class ExternalSourcesItemModel {
public String getUri() {
return uri;
}
public void setUri(String uri) {
this.uri = uri;
}
@ -46,7 +50,6 @@ public class ExternalSourcesItemModel {
public String getRemoteId() {
return remoteId;
}
public void setRemoteId(String remoteId) {
this.remoteId = remoteId;
}
@ -54,7 +57,6 @@ public class ExternalSourcesItemModel {
public String getAbbreviation() {
return abbreviation;
}
public void setAbbreviation(String abbreviation) {
this.abbreviation = abbreviation;
}
@ -62,7 +64,6 @@ public class ExternalSourcesItemModel {
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
@ -70,7 +71,6 @@ public class ExternalSourcesItemModel {
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}

74
dmp-backend/web/src/main/java/eu/eudat/models/data/externaldataset/ExternalDatasetListingModel.java
View File

@ -10,34 +10,35 @@ import java.util.UUID;
public class ExternalDatasetListingModel implements DataModel<ExternalDataset, ExternalDatasetListingModel> {
private UUID id;
private String label;
private String name;
private String abbreviation;
private String reference;
private Date created;
private Date modified;
private String info;
private ExternalDatasetType type;
private String pid;
private String uri;
private String tag; // Api fetching the data
private String source; // Actual harvested source
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public String getLabel() {
return label;
public String getName() {
return name;
}
public void setLabel(String label) {
this.label = label;
public void setName(String name) {
this.name = name;
}
public String getAbbreviation() {
return abbreviation;
}
public void setAbbreviation(String abbreviation) {
this.abbreviation = abbreviation;
}
@ -45,7 +46,6 @@ public class ExternalDatasetListingModel implements DataModel<ExternalDataset, E
public String getReference() {
return reference;
}
public void setReference(String reference) {
this.reference = reference;
}
@ -53,7 +53,6 @@ public class ExternalDatasetListingModel implements DataModel<ExternalDataset, E
public Date getCreated() {
return created;
}
public void setCreated(Date created) {
this.created = created;
}
@ -61,7 +60,6 @@ public class ExternalDatasetListingModel implements DataModel<ExternalDataset, E
public Date getModified() {
return modified;
}
public void setModified(Date modified) {
this.modified = modified;
}
@ -69,7 +67,6 @@ public class ExternalDatasetListingModel implements DataModel<ExternalDataset, E
public Integer getType() {
return type != null ? type.getValue() : null;
}
public void setType(Integer type) {
this.type = ExternalDatasetType.fromInteger(type);
}
@ -77,19 +74,52 @@ public class ExternalDatasetListingModel implements DataModel<ExternalDataset, E
public String getInfo() {
return info;
}
public void setInfo(String info) {
this.info = info;
}
public String getPid() {
return pid;
}
public void setPid(String pid) {
this.pid = pid;
}
public String getUri() {
return uri;
}
public void setUri(String uri) {
this.uri = uri;
}
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
@Override
public ExternalDatasetListingModel fromDataModel(ExternalDataset entity) {
this.id = entity.getId();
this.abbreviation = entity.getAbbreviation();
this.label = entity.getLabel();
this.name = entity.getLabel();
this.modified = entity.getModified();
this.created = entity.getCreated();
this.reference = entity.getReference();
String source1 = entity.getReference().substring(0, entity.getReference().indexOf(":"));
if (source1.equals("dmp")) {
this.source = "Internal";
} else {
this.source = source1;
}
return this;
}
@ -98,10 +128,22 @@ public class ExternalDatasetListingModel implements DataModel<ExternalDataset, E
ExternalDataset externalDataset = new ExternalDataset();
externalDataset.setAbbreviation(this.abbreviation);
externalDataset.setCreated(this.created != null ? this.created : new Date());
externalDataset.setLabel(this.label);
externalDataset.setLabel(this.name);
externalDataset.setId(this.id);
externalDataset.setModified(this.modified);
externalDataset.setReference(this.reference);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
externalDataset.setReference(this.reference);
} else {
externalDataset.setReference(this.source + ":" + this.reference);
}
}
if (this.reference == null) {
externalDataset.setReference(this.source + ":" + this.name);
}
externalDataset.setModified(new Date());
return externalDataset;
}

2
dmp-frontend/src/app/core/model/external-dataset/external-dataset.ts
View File

@ -3,7 +3,7 @@ import { ExternalDatasetType } from '../../common/enum/external-dataset-type';
export interface ExternalDatasetModel {
abbreviation: String;
id: String;
label: String;
name: String;
reference: String;
type: ExternalDatasetType;
info: String;

10
dmp-frontend/src/app/ui/dataset/dataset-wizard/dataset-wizard-editor.model.ts
View File

@ -260,16 +260,16 @@ export class ExternalDatasetEditorModel {
public abbreviation: String;
public id: String;
public label: String;
public name: String;
public reference: String;
public type: ExternalDatasetType;
public info: String;
public validationErrorModel: ValidationErrorModel = new ValidationErrorModel();
public source: String;
constructor(id?: string, abbreviation?: string, label?: string, reference?: string, info?: string, type?: ExternalDatasetType, source?: String) {
constructor(id?: string, abbreviation?: string, name?: string, reference?: string, source?: String, info?: string, type?: ExternalDatasetType) {
this.id = id;
this.label = label;
this.name = name;
this.abbreviation = abbreviation;
this.reference = reference;
this.info = info;
@ -280,7 +280,7 @@ export class ExternalDatasetEditorModel {
fromModel(item: ExternalDatasetModel): ExternalDatasetEditorModel {
this.abbreviation = item.abbreviation;
this.id = item.id;
this.label = item.label;
this.name = item.name;
this.reference = item.reference;
this.type = item.type;
this.info = item.info;
@ -292,7 +292,7 @@ export class ExternalDatasetEditorModel {
return new FormBuilder().group({
id: [this.id],
abbreviation: [this.abbreviation],
label: [this.label],
name: [this.name],
reference: [this.reference],
type: [this.type, Validators.required],
info: [this.info],

2
dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/dataset-external-references-editor.component.html
View File

@ -52,7 +52,7 @@
<div class="col-12 row align-items-center">
<div class="col">
<p>
{{i+1}}) {{suggestion.get('label').value}}
{{i+1}}) {{suggestion.get('name').value}}
</p>
</div>
<div class="col-auto">

10
dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/dataset-external-references-editor.component.ts
View File

@ -77,9 +77,9 @@ export class DatasetExternalReferencesEditorComponent extends BaseComponent impl
filterFn: this.searchDatasetExternalDatasets.bind(this),
removeAfterSelection: true,
initialItems: (type) => this.searchDatasetExternalDatasets('', type),//.filter(resultItem => excludedItems.map(x => x.id).indexOf(resultItem.id) === -1),
displayFn: (item) => item ? item.label : null,
titleFn: (item) => item ? item.label : null,
subtitleFn: (item) => item.tag ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.tag : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE')
displayFn: (item) => item ? item.name : null,
titleFn: (item) => item ? item.name : null,
subtitleFn: (item) => item.source ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.source : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE')
};
this.registriesAutoCompleteConfiguration = {
@ -97,7 +97,7 @@ export class DatasetExternalReferencesEditorComponent extends BaseComponent impl
initialItems: (type) => this.searchDatasetExternalServices('', type),
displayFn: (item) => item ? item.label : null,
titleFn: (item) => item ? item.label : null,
subtitleFn: (item) => item.tag ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.tag : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE')
subtitleFn: (item) => item.source ? this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.SOURCE:') + item.source : this.language.instant('TYPES.EXTERNAL-DATASET-TYPE.NO-SOURCE')
};
this.tagsAutoCompleteConfiguration = {
@ -115,7 +115,7 @@ export class DatasetExternalReferencesEditorComponent extends BaseComponent impl
}
externalDatasetsOnItemChange(event) {
const externalDatasetModel = new ExternalDatasetEditorModel(event.id, event.abbreviation, event.label, event.reference, event.source);
const externalDatasetModel = new ExternalDatasetEditorModel(event.id, event.abbreviation, event.name, event.pid ? event.pid : event.reference, event.source);
(<FormArray>this.formGroup.get('externalDatasets')).push(externalDatasetModel.buildForm());
}

4
dmp-frontend/src/app/ui/dataset/dataset-wizard/external-references/editors/external-dataset/dataset-external-dataset-dialog-editor.component.html
View File

@ -9,8 +9,8 @@
</div>
<div mat-dialog-content class="row">
<mat-form-field class="col-auto">
<input matInput formControlName="label" placeholder="{{'DATASET-REFERENCED-MODELS.EXTERNAL-DATASET.LABEL' | translate}}" required>
<mat-error *ngIf="formGroup.get('label').hasError('required')">{{'GENERAL.VALIDATION.REQUIRED' | translate}}</mat-error>
<input matInput formControlName="name" placeholder="{{'DATASET-REFERENCED-MODELS.EXTERNAL-DATASET.LABEL' | translate}}" required>
<mat-error *ngIf="formGroup.get('name').hasError('required')">{{'GENERAL.VALIDATION.REQUIRED' | translate}}</mat-error>
</mat-form-field>
<mat-form-field class="col-auto">
<input matInput formControlName="abbreviation" placeholder="{{'DATASET-REFERENCED-MODELS.EXTERNAL-DATASET.ABBREVIATION' | translate}}" required>