Merge branch 'dmp-refactoring' of https://code-repo.d4science.org/MaDgiK-CITE/argos into dmp-refactoring

This commit is contained in:
Diamantis Tziotzios 2023-11-16 15:41:01 +02:00
commit df58a254b2
8 changed files with 146 additions and 46 deletions

View File

@ -29,6 +29,9 @@ public class ReferenceType {
private Instant updatedAt;
public static final String _updatedAt = "updatedAt";
public final static String _hash = "hash";
private String hash;
public UUID getId() {
return id;
}
@ -84,4 +87,12 @@ public class ReferenceType {
public void setUpdatedAt(Instant updatedAt) {
this.updatedAt = updatedAt;
}
public String getHash() {
return hash;
}
public void setHash(String hash) {
this.hash = hash;
}
}

View File

@ -2,14 +2,11 @@ package eu.eudat.model.builder;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.ReferenceType;
import eu.eudat.model.builder.referencedefinition.DefinitionBuilder;
import eu.eudat.model.builder.referencetypedefinition.ReferenceTypeDefinitionBuilder;
import eu.eudat.model.referencetypedefinition.ReferenceTypeDefinition;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
@ -66,6 +63,7 @@ public class ReferenceTypeBuilder extends BaseBuilder<ReferenceType, ReferenceTy
if (fields.hasField(this.asIndexer(ReferenceType._createdAt))) m.setCreatedAt(d.getCreatedAt());
if (fields.hasField(this.asIndexer(ReferenceType._updatedAt))) m.setUpdatedAt(d.getUpdatedAt());
if (fields.hasField(this.asIndexer(ReferenceType._isActive))) m.setIsActive(d.getIsActive());
if (fields.hasField(this.asIndexer(ReferenceType._hash))) m.setHash(this.hashValue(d.getUpdatedAt()));
if (!definitionFields.isEmpty() && d.getDefinition() != null){
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, d.getDefinition());
m.setDefinition(this.builderFactory.builder(ReferenceTypeDefinitionBuilder.class).authorize(this.authorize).build(definitionFields, definition));

View File

@ -8,6 +8,7 @@ import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.types.referencetype.*;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.errorcode.ErrorThesaurusProperties;
import eu.eudat.model.ReferenceType;
import eu.eudat.model.builder.ReferenceTypeBuilder;
import eu.eudat.model.deleter.ReferenceTypeDeleter;
@ -53,12 +54,13 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
private final MessageSource messageSource;
private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService;
private final ErrorThesaurusProperties errors;
public ReferenceTypeServiceImpl(
EntityManager entityManager, AuthorizationService authorizationService, DeleterFactory deleterFactory, BuilderFactory builderFactory,
ConventionService conventionService, MessageSource messageSource, QueryFactory queryFactory,
XmlHandlingService xmlHandlingService) {
XmlHandlingService xmlHandlingService, ErrorThesaurusProperties errors) {
this.entityManager = entityManager;
this.authorizationService = authorizationService;
this.deleterFactory = deleterFactory;
@ -67,6 +69,7 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
this.messageSource = messageSource;
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
this.errors = errors;
}
@ -82,6 +85,7 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
data = this.entityManager.find(ReferenceTypeEntity.class, model.getId());
if (data == null)
throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
} else {
data = new ReferenceTypeEntity();

View File

@ -292,12 +292,12 @@ public class ReferenceService {
return null;
}
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources());
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources(), lookup);
return null;
}
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources){
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
List<Map<String, String>> results = new LinkedList<>();
if (sources == null || sources.isEmpty()) {
@ -305,19 +305,29 @@ public class ReferenceService {
}
sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
apiSources.forEach(source -> {
try {
String auth = null;
if (source.getAuth()!= null) {
//auth = this.getAuthentication(source.getAuth());
}
results.addAll(getAllResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), null, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
});
List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
staticSources.forEach(source -> {
Map<String, String> map = new HashMap<>();
source.getOptions().forEach(option -> {
map.put(option.getCode(), option.getValue());
map.put("label", source.getLabel());
map.put("key", source.getKey());
});
results.add(map);
});
return results;
}
@ -349,23 +359,21 @@ public class ReferenceService {
return finalBodyString;
}
private List<Map<String, String>> getAllResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity jsonResultsPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
Set<Integer> pages = new HashSet<>();
//String replacedPath = replaceCriteriaOnUrl(urlPath, externalUrlCriteria, firstPage, queries);
String replacedUrlPath = urlPath;
//String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage, queries);
String replacedUrlBody = requestBody;
String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
Results results = getResultsFromUrl(replacedUrlPath, jsonResultsPath, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
Results results = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
if(results != null) {
if (filterType != null && filterType.equals("local") && (externalUrlCriteria.getLike() != null && !externalUrlCriteria.getLike().isEmpty())) {
if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
results.setResults(results.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(externalUrlCriteria.getLike().toLowerCase()))
.filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
if (fetchStrategy == FetchStrategy.FIRST)
return results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
return results.getResults().stream().peek(x -> x.put("label", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= results.getPagination().get("pages"); i++)
@ -374,10 +382,10 @@ public class ReferenceService {
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
Long maxResults = Long.valueOf(1000);
if ((maxResults > 0) && (results.getPagination().get("count") > maxResults))
throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
throw new HugeResultSet("The submitted search query " + lookup.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, jsonResultsPath, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
.filter(Objects::nonNull)
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
@ -386,14 +394,52 @@ public class ReferenceService {
Results remainingResults = optionalResults.orElseGet(Results::new);
remainingResults.getResults().addAll(results.getResults());
return remainingResults.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
return remainingResults.getResults().stream().peek(x -> x.put("label", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
}
else {
return new LinkedList<>();
}
}
protected Results getResultsFromUrl(String urlString, ResultsConfigurationEntity jsonResultsPath, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
String completedPath = urlPath;
if (urlPath.contains("{like}")){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "*");
}
}
if (urlPath.contains("{page}")){
if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
} else {
completedPath = completedPath.replace("{page}", "1");
}
}
if (urlPath.contains("{pageSize}")){
if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
} else {
completedPath = completedPath.replace("{pageSize}", "100");
}
}
if (urlPath.contains("openaire")){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{query}", lookup.getLike());
} else {
completedPath = completedPath.replace("{query}", "*");
}
}
return completedPath;
}
protected Results getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
try {
ResponseEntity<String> response;
@ -412,7 +458,7 @@ public class ReferenceService {
Results results = new Results();
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
results = this.getFromJson(jsonContext, jsonResultsPath);
results = this.getFromJson(jsonContext, resultsEntity);
}
@ -430,24 +476,24 @@ public class ReferenceService {
}
public static Results getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity jsonResultsPath) {
return new Results(parseData(jsonContext, jsonResultsPath),
public static Results getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
return new Results(parseData(jsonContext, resultsEntity),
new HashMap<>(1, 1));
}
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity jsonResultsPath) {
List <Map<String, String>> rawData = jsonContext.read(jsonResultsPath.getResultsArrayPath());
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
List<Map<String, String>> parsedData = new ArrayList<>();
rawData.forEach(stringObjectMap -> {
Map<String, String> map = new HashMap<>();
jsonResultsPath.getFieldsMapping().forEach(field ->{
resultsEntity.getFieldsMapping().forEach(field ->{
String pathValue = field.getResponsePath();
if (stringObjectMap.containsKey(pathValue)){
map.put(field.getCode(), stringObjectMap.get(pathValue));
parsedData.add(map);
}
});
parsedData.add(map);
});

View File

@ -132,6 +132,8 @@
</mat-card-content>
</div>
</div>
<button mat-button class="action-btn" *ngIf="formGroup.get('definition').get('fields').value != ''"
type="button" (click)="submitFields()" [disabled]="!formGroup.get('definition').get('fields').valid">Submit</button>
</div>
</form>
</mat-card-content>
@ -146,7 +148,7 @@
</mat-card-header>
<mat-card-content>
<form (ngSubmit)="formSubmit()" [formGroup]="formGroup" *ngIf="formGroup">
<!-- External Api Config Info -->
<!-- Source Config Info -->
<div class="col-12">
<div *ngFor="let source of formGroup.get('definition').get('sources').controls; let sourceIndex=index;" class="row mb-3" cdkDrag [cdkDragDisabled]="formGroup.disabled">
<div class="col-12">

View File

@ -117,7 +117,9 @@ export class ReferenceTypeEditorComponent extends BaseEditor<ReferenceTypeEditor
ngOnInit(): void {
super.ngOnInit();
//this.addSource();
if((this.formGroup.get('definition').get('sources') as FormArray).length == 0){
this.addSource();
}
}
getItem(itemId: Guid, successFunction: (item: ReferenceType) => void) {
@ -225,14 +227,6 @@ export class ReferenceTypeEditorComponent extends BaseEditor<ReferenceTypeEditor
// }
// const sourceSize = (this.formGroup.get('definition').get('sources') as FormArray).length;
// if(sourceSize && sourceSize > 0){
// for(let i =0; i < sourceSize; i++){
// this.addFieldMapping(i);
// this.addOption(i);
// }
// }
}
@ -247,21 +241,36 @@ export class ReferenceTypeEditorComponent extends BaseEditor<ReferenceTypeEditor
fieldssFormArray.updateValueAndValidity();
}
submitFields(): void{
const fieldssFormArray = (this.formGroup.get('definition').get('fields') as FormArray);
if (fieldssFormArray.valid){
const sourceSize = (this.formGroup.get('definition').get('sources') as FormArray).length;
if(sourceSize && sourceSize > 0){
for(let i =0; i < sourceSize; i++){
this.addFieldMapping(i, fieldssFormArray.at(i).get('code').value);
this.addOption(i, fieldssFormArray.at(i).get('code').value);
}
}
}
}
addSource(): void{
const source: ReferenceTypeSourceBaseConfigurationEditorModel = new ReferenceTypeSourceBaseConfigurationEditorModel();
(this.formGroup.get('definition').get('sources') as FormArray).push(source.buildForm());
const sourceIndex = (this.formGroup.get('definition').get('sources') as FormArray).length - 1;
this.systemFieldsMapping.forEach(x => {
this.addFieldMapping(sourceIndex);
this.addOption(sourceIndex);
this.addFieldMapping(sourceIndex, null);
this.addOption(sourceIndex, null);
});
const fieldsSize = (this.formGroup.get('definition').get('fields') as FormArray).length;
if(fieldsSize && fieldsSize > 0){
for(let i =0; i < fieldsSize; i++){
this.addFieldMapping(sourceIndex);
this.addOption(sourceIndex);
this.addFieldMapping(sourceIndex, null);
this.addOption(sourceIndex, null);
}
}
}
@ -275,9 +284,23 @@ export class ReferenceTypeEditorComponent extends BaseEditor<ReferenceTypeEditor
// resultFieldsMapping
//
//
addFieldMapping(sourceIndex: number): void {
addFieldMapping(sourceIndex: number, code: string): void {
const fieldMapping: ResultFieldsMappingConfigurationEditorModel = new ResultFieldsMappingConfigurationEditorModel();
//((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('results').get('fieldsMapping') as FormArray).push(fieldMapping.buildForm());
const fieldMappingSize = ((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('results').get('fieldsMapping') as FormArray).length;
if (fieldMappingSize>0){
for(let i=0; i<fieldMappingSize; i++){
if(((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('results').get('fieldsMapping') as FormArray).at(i).get('code').value == code){
console.log('error');
return;
};
}
}
((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('results').get('fieldsMapping') as FormArray).push(fieldMapping.buildForm());
((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('results').get('fieldsMapping') as FormArray).at(fieldMappingSize).get('code').setValue('1');
}
removeFieldMapping(sourceIndex: number, fieldMappingIndex: number): void {
@ -317,9 +340,21 @@ export class ReferenceTypeEditorComponent extends BaseEditor<ReferenceTypeEditor
// Options
addOption(sourceIndex: number): void {
addOption(sourceIndex: number, code: string): void {
const options: ReferenceTypeStaticOptionEditorModel = new ReferenceTypeStaticOptionEditorModel();
const optionsSize = ((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('options') as FormArray).length;
if (optionsSize>0){
for(let i=0; i<optionsSize; i++){
if(((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('options') as FormArray).at(i).get('code').value == code){
console.log('error');
return;
};
}
}
((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('options') as FormArray).push(options.buildForm());
((this.formGroup.get('definition').get('sources') as FormArray).at(sourceIndex).get('options') as FormArray).at(optionsSize).get('code').setValue('1');
}
}

View File

@ -42,7 +42,8 @@ export class ReferenceTypeEditorModel extends BaseEditorModel implements Referen
code: [{ value: this.code, disabled: disabled }, context.getValidation('code').validators],
definition: this.definition.buildForm({
rootPath: `definition.`
})
}),
hash: [{ value: this.hash, disabled: disabled }, context.getValidation('hash').validators]
});
}
@ -52,6 +53,7 @@ export class ReferenceTypeEditorModel extends BaseEditorModel implements Referen
baseValidationArray.push({ key: 'id', validators: [BackendErrorValidator(this.validationErrorModel, 'id')] });
baseValidationArray.push({ key: 'name', validators: [Validators.required, BackendErrorValidator(this.validationErrorModel, 'name')] });
baseValidationArray.push({ key: 'code', validators: [Validators.required, BackendErrorValidator(this.validationErrorModel, 'code')] });
baseValidationArray.push({ key: 'hash', validators: [] });
baseContext.validation = baseValidationArray;
return baseContext;

View File

@ -1046,9 +1046,11 @@ export class DmpEditorBlueprintComponent extends CheckDeactivateBaseComponent im
// lookup.type = ReferenceType.Researcher;
// return this.referenceService.search(lookup);
const lookup = new ReferenceSearchDefinitionLookup();
//lookup.like='';
//lookup.page.size = 1;
lookup.key = '';
//from reference type db hardcoded
lookup.referenceTypeId = Guid.parse('c8400d41-28f1-477a-8fa1-3351876fca4f');
lookup.referenceTypeId = Guid.parse('51225b6a-86a6-48ac-9192-f15096dbcb8a');
const fields = [
...ReferenceTypeEditorResolver.lookupFields()
];