MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

RemoteFetcherService changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-13 12:22:11 +02:00
parent 28d08d99e8
commit 40316781a2
4 changed files with 57 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dmp-backend/core/src/main/java/eu/eudat/data/ReferenceEntity.java
View File

@ -14,6 +14,16 @@ import java.util.UUID;
@Table(name = "\"Reference\"") @Table(name = "\"Reference\"")
public class ReferenceEntity extends TenantScopedBaseEntity { public class ReferenceEntity extends TenantScopedBaseEntity {
public static class KnownFields {
public static final String Key = "key";
public static final String ReferenceId = "reference_id";
public static final String Abbreviation = "abbreviation";
public static final String Description = "description";
public static final String Label = "label";
public static final String SourceLabel = "tag";
}
@Id @Id
@Column(name = "id", columnDefinition = "uuid", updatable = false, nullable = false) @Column(name = "id", columnDefinition = "uuid", updatable = false, nullable = false)
private UUID id; private UUID id;

12
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java
View File

@ -201,14 +201,14 @@ public class ReferenceServiceImpl implements ReferenceService {
@NotNull @NotNull
private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) { private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) {
ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic ReferenceEntity referenceEntity = new ReferenceEntity();
referenceEntity.setTypeId(data.getId()); referenceEntity.setTypeId(data.getId());
referenceEntity.setIsActive(IsActive.Active); referenceEntity.setIsActive(IsActive.Active);
referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null)); referenceEntity.setReference(result.getOrDefault(ReferenceEntity.KnownFields.Key, null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("reference_id", null));
referenceEntity.setSource(result.getOrDefault("tag", null)); referenceEntity.setSource(result.getOrDefault(ReferenceEntity.KnownFields.Key, null));
referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null)); referenceEntity.setAbbreviation(result.getOrDefault(ReferenceEntity.KnownFields.Abbreviation, null));
referenceEntity.setDescription(result.getOrDefault("description", null)); referenceEntity.setDescription(result.getOrDefault(ReferenceEntity.KnownFields.Description, null));
referenceEntity.setLabel(result.getOrDefault("name", null)); referenceEntity.setLabel(result.getOrDefault(ReferenceEntity.KnownFields.Abbreviation, null));
referenceEntity.setSourceType(ReferenceSourceType.External); referenceEntity.setSourceType(ReferenceSourceType.External);
DefinitionEntity definitionEntity = new DefinitionEntity(); DefinitionEntity definitionEntity = new DefinitionEntity();
definitionEntity.setFields(new ArrayList<>()); definitionEntity.setFields(new ArrayList<>());

117
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java
View File

@ -5,10 +5,10 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPath;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.ReferenceTypeSourceType; import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.convention.ConventionService; import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.models.ExternalDataResult; import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
@ -26,8 +26,6 @@ import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient; import reactor.netty.http.client.HttpClient;
import java.io.File;
import java.nio.file.Paths;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -38,12 +36,10 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
private WebClient webClient; private WebClient webClient;
private final ExternalUrlConfigProvider externalUrlConfigProvider; private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final ConventionService conventionService; private final ConventionService conventionService;
private final XmlHandlingService xmlHandlingService;
@Autowired @Autowired
public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) { public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService) {
this.externalUrlConfigProvider = externalUrlConfigProvider; this.externalUrlConfigProvider = externalUrlConfigProvider;
this.conventionService = conventionService; this.conventionService = conventionService;
this.xmlHandlingService = xmlHandlingService;
} }
private WebClient getWebClient(){ private WebClient getWebClient(){
@ -61,9 +57,9 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
@Override @Override
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) { public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
List<SourceBaseConfiguration> apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList(); List<SourceBaseConfiguration> apiSourcesToUse = sources;
if (!this.conventionService.isNullOrEmpty(key)){ if (!this.conventionService.isNullOrEmpty(key)){
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList(); apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList();
} }
if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult(); if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
@ -99,11 +95,28 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
} }
} else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) { } else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) {
SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source; SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO: results.addAll(queryStaticData(staticSource, externalReferenceCriteria));
} }
} }
return results; return results;
} }
private ExternalDataResult queryStaticData(SourceStaticOptionConfiguration<StaticOption> staticSource, ExternalReferenceCriteria externalReferenceCriteria){
Map<String, String> result = new HashMap<>();
Map<String, Object> rawData = new HashMap<>();
for (StaticOption staticOption : staticSource.getOptions()){
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike()) && !externalReferenceCriteria.getLike().toUpperCase().contains(staticOption.getValue())) continue;
result.put(staticOption.getCode(), staticOption.getValue());
rawData.put(staticOption.getCode(), staticOption.getValue());
result.put(ReferenceEntity.KnownFields.SourceLabel, staticSource.getLabel());
result.put(ReferenceEntity.KnownFields.Key, staticSource.getKey());
}
ExternalDataResult externalDataResult = new ExternalDataResult();
externalDataResult.setRawData(new ArrayList<>());
externalDataResult.setResults(new ArrayList<>());
externalDataResult.getRawData().add(rawData);
externalDataResult.getResults().add(result);
return externalDataResult;
}
private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) { private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
HttpMethod method; HttpMethod method;
@ -210,7 +223,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
.filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
.collect(Collectors.toList())); .collect(Collectors.toList()));
} }
externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList()); externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put(ReferenceEntity.KnownFields.SourceLabel, apiSource.getLabel())).peek(x -> x.put(ReferenceEntity.KnownFields.Key, apiSource.getKey())).toList());
if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult; if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
@ -252,7 +265,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
if (responseContentType.contains("json") ) { if (responseContentType.contains("json") ) {
DocumentContext jsonContext = JsonPath.parse(response.getBody()); DocumentContext jsonContext = JsonPath.parse(response.getBody());
return this.parseData(jsonContext, apiSource.getResults()); return this.jsonToExternalDataResult(jsonContext, apiSource.getResults());
} else { } else {
throw new MyApplicationException("Unsupported response type" + responseContentType); throw new MyApplicationException("Unsupported response type" + responseContentType);
} }
@ -264,60 +277,35 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
return null; return null;
} }
private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) { private ExternalDataResult jsonToExternalDataResult(DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
ExternalDataResult result = new ExternalDataResult(); ExternalDataResult result = new ExternalDataResult();
if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult(); if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath()); List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
result.setRawData(rawData); result.setRawData(rawData);
if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult(); if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
List<Object> results = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
List<Map<String, String>> parsedData = new ArrayList<>(); List<Map<String, String>> parsedData = new ArrayList<>();
for (Map<String, Object> stringObjectMap: rawData){ for(Object resultItem : results){
Map<String, String> map = new HashMap<>(); Map<String, String> map = new HashMap<>();
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){ for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
String pathValue = field.getResponsePath(); StringBuilder pathValue = new StringBuilder();
if (!pathValue.contains(".")){ for (String pathPart : field.getResponsePath().split("\\.")) {
if (stringObjectMap.containsKey(pathValue)) { pathValue.append("['").append(pathPart).append("']");
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
}
}else {
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
String value = null;
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
if (fieldObj != null){
if (fieldObj instanceof Map){
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
} else if (fieldObj instanceof List) {
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
}
}
if (value != null){
map.put(field.getCode(), value);
}
}
} }
Object value = JsonPath.parse(resultItem).read(pathValue.toString());
map.put(field.getCode(), normalizeJsonValue(value));
} }
parsedData.add(map); parsedData.add(map);
} }
result.setResults(parsedData); result.setResults(parsedData);
return result; return result;
} }
private static String normalizeValue(Object value) { private static String normalizeJsonValue(Object value) {
if (value instanceof JSONArray jsonArray) { if (value instanceof JSONArray jsonArray) {
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) { if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
@ -326,7 +314,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
for (Object o : jsonArray) { for (Object o : jsonArray) {
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) { if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
try { try {
return ((Map<String, String>) o).get("content"); return String.valueOf(((Map<?, ?>) o).get("content"));
} }
catch (ClassCastException e){ catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) { if(((Map<?, ?>) o).get("content") instanceof Integer) {
@ -338,39 +326,12 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
} }
} }
} else if (value instanceof Map) { } else if (value instanceof Map) {
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content"; String key = ((Map<?, ?>)value).containsKey("$") ? "$" : "content";
return ((Map<String, String>)value).get(key); return String.valueOf(((Map<?, ?>)value).get(key));
} }
return value != null ? value.toString() : null; return value != null ? value.toString() : null;
} }
private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
List<Map<String, String>> internalResults;
try {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
return new ExternalDataResult(searchListMap(internalResults, query));
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new ExternalDataResult();
}
}
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
List<Map<String, String>> list = new LinkedList<>();
for (Map<String, String> map : internalResults)
{
if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
}
return list;
}
private String parseBodyString(String bodyString) { private String parseBodyString(String bodyString) {
String finalBodyString = bodyString; String finalBodyString = bodyString;
if (bodyString.contains("{env:")) { if (bodyString.contains("{env:")) {

2
dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java
View File

@ -12,6 +12,8 @@ import org.springframework.context.annotation.Primary;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableAsync;
import java.util.Map;
@SpringBootApplication(scanBasePackages = { @SpringBootApplication(scanBasePackages = {
"eu.eudat", "eu.eudat",
"eu.eudat.depositinterface", "eu.eudat.depositinterface",