external fetcher fixes add first logic

This commit is contained in:
Efstratios Giannopoulos 2024-05-01 13:46:57 +03:00
parent 81156955dc
commit 1474eda8b9
1 changed files with 28 additions and 43 deletions

View File

@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import gr.cite.tools.exception.MyApplicationException;
import net.minidev.json.JSONArray;
import org.opencdmp.commons.JsonHandlingService;
import org.opencdmp.commons.enums.ExternalFetcherSourceType;
import org.opencdmp.commons.types.externalfetcher.StaticOptionEntity;
@ -13,25 +15,22 @@ import org.opencdmp.data.ReferenceEntity;
import org.opencdmp.model.Reference;
import org.opencdmp.model.referencedefinition.Field;
import org.opencdmp.service.externalfetcher.config.entities.*;
import org.opencdmp.service.externalfetcher.models.ExternalDataResult;
import org.opencdmp.service.externalfetcher.criteria.ExternalReferenceCriteria;
import gr.cite.tools.exception.MyApplicationException;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
import net.minidev.json.JSONArray;
import org.opencdmp.service.externalfetcher.models.ExternalDataResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import javax.net.ssl.SSLException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -58,7 +57,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
}
return webClient;
return this.webClient;
}
@ -78,7 +77,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
@Override
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
return getExternalData(sources, externalReferenceCriteria, key).getResults().size();
return this.getExternalData(sources, externalReferenceCriteria, key).getResults().size();
}
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria) {
@ -103,7 +102,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
}
} else if (source.getType() != null && source.getType().equals(ExternalFetcherSourceType.STATIC)) {
SourceStaticOptionConfiguration<Static> staticSource = (SourceStaticOptionConfiguration)source;
results.addAll(queryStaticData(staticSource, externalReferenceCriteria));
results.addAll(this.queryStaticData(staticSource, externalReferenceCriteria));
}
}
return results;
@ -232,12 +231,12 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
}
private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
String replacedPath = replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
String replacedPath = this.replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
String replacedBody = this.replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
if(externalDataResult != null) {
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
if (apiSource.getFilterType() != null && "local".equals(apiSource.getFilterType()) && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
externalDataResult.setResults(externalDataResult.getResults().stream()
.filter(r -> r.get(ReferenceEntity.KnownFields.Label).toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
.collect(Collectors.toList()));
@ -310,28 +309,30 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
boolean isValid = true;
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue;
boolean getFirst = field.getResponsePath().endsWith(".first()");
String responsePath = new String(field.getResponsePath());
if (getFirst) responsePath = responsePath.substring(0, responsePath.length() - ".first()".length());
try {
if (field.getResponsePath().contains("@{{")){
if (responsePath.contains("@{{")){
String rePattern = "@\\{\\{(.*?)}}";
Pattern p = Pattern.compile(rePattern);
Matcher m = p.matcher(field.getResponsePath());
String value = field.getResponsePath();
Matcher m = p.matcher(responsePath);
String value = responsePath;
while (m.find()) {
if (m.groupCount() < 1) continue;
Object partValue = JsonPath.parse(resultItem).read(m.group(1));
String normalizedValue = normalizeJsonValue(partValue);
String normalizedValue = normalizeJsonValue(partValue, getFirst);
if (normalizedValue != null) value = value.replace("@{{" + m.group(1) + "}}", normalizedValue);
else value = value.replace("@{{" + m.group(1) + "}}","");
}
map.put(field.getCode(), normalizeJsonValue(value));
map.put(field.getCode(), normalizeJsonValue(value, getFirst));
} else {
Object value = JsonPath.parse(resultItem).read(field.getResponsePath());
map.put(field.getCode(), normalizeJsonValue(value));
Object value = JsonPath.parse(resultItem).read(responsePath);
map.put(field.getCode(), normalizeJsonValue(value, getFirst));
}
}catch (PathNotFoundException e){
logger.debug("Json Path Error: " + e.getMessage() + " on source " + jsonHandlingService.toJsonSafe(resultItem));
logger.debug("Json Path Error: " + e.getMessage() + " on source " + this.jsonHandlingService.toJsonSafe(resultItem));
if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) {
isValid = false;
break;
@ -339,7 +340,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
}
}
if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){
logger.warn("Invalid reference on source " + jsonHandlingService.toJsonSafe(resultItem));
logger.warn("Invalid reference on source " + this.jsonHandlingService.toJsonSafe(resultItem));
}
if (isValid) parsedData.add(map);
}
@ -348,29 +349,13 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
return result;
}
private static String normalizeJsonValue(Object value) {
private static String normalizeJsonValue(Object value, boolean getfirst) {
if (value instanceof JSONArray jsonArray) {
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
if (getfirst) {
return jsonArray.getFirst().toString();
} else {
for (Object o : jsonArray) {
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
try {
return String.valueOf(((Map<?, ?>) o).get("content"));
return value.toString();
}
catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) {
return String.valueOf(((Map<?, ?>) o).get("content"));
}
return null;
}
}
}
}
} else if (value instanceof Map) {
String key = ((Map<?, ?>)value).containsKey("$") ? "$" : "content";
return String.valueOf(((Map<?, ?>)value).get(key));
}
return value != null ? value.toString() : null;
}