external fetcher fixes add first logic
This commit is contained in:
parent
81156955dc
commit
1474eda8b9
|
|
@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||
import com.jayway.jsonpath.DocumentContext;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import com.jayway.jsonpath.PathNotFoundException;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import net.minidev.json.JSONArray;
|
||||
import org.opencdmp.commons.JsonHandlingService;
|
||||
import org.opencdmp.commons.enums.ExternalFetcherSourceType;
|
||||
import org.opencdmp.commons.types.externalfetcher.StaticOptionEntity;
|
||||
|
|
@ -13,25 +15,22 @@ import org.opencdmp.data.ReferenceEntity;
|
|||
import org.opencdmp.model.Reference;
|
||||
import org.opencdmp.model.referencedefinition.Field;
|
||||
import org.opencdmp.service.externalfetcher.config.entities.*;
|
||||
import org.opencdmp.service.externalfetcher.models.ExternalDataResult;
|
||||
import org.opencdmp.service.externalfetcher.criteria.ExternalReferenceCriteria;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import io.netty.handler.ssl.SslContextBuilder;
|
||||
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
|
||||
import net.minidev.json.JSONArray;
|
||||
import org.opencdmp.service.externalfetcher.models.ExternalDataResult;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||
import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import reactor.netty.http.client.HttpClient;
|
||||
|
||||
import javax.net.ssl.SSLException;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
|
@ -58,7 +57,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
}
|
||||
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
|
||||
}
|
||||
return webClient;
|
||||
return this.webClient;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -78,7 +77,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
|
||||
@Override
|
||||
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
|
||||
return getExternalData(sources, externalReferenceCriteria, key).getResults().size();
|
||||
return this.getExternalData(sources, externalReferenceCriteria, key).getResults().size();
|
||||
}
|
||||
|
||||
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||
|
|
@ -103,7 +102,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
}
|
||||
} else if (source.getType() != null && source.getType().equals(ExternalFetcherSourceType.STATIC)) {
|
||||
SourceStaticOptionConfiguration<Static> staticSource = (SourceStaticOptionConfiguration)source;
|
||||
results.addAll(queryStaticData(staticSource, externalReferenceCriteria));
|
||||
results.addAll(this.queryStaticData(staticSource, externalReferenceCriteria));
|
||||
}
|
||||
}
|
||||
return results;
|
||||
|
|
@ -232,12 +231,12 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
}
|
||||
|
||||
private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
|
||||
String replacedPath = replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
|
||||
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
|
||||
String replacedPath = this.replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
|
||||
String replacedBody = this.replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
|
||||
|
||||
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
|
||||
if(externalDataResult != null) {
|
||||
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
|
||||
if (apiSource.getFilterType() != null && "local".equals(apiSource.getFilterType()) && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
|
||||
externalDataResult.setResults(externalDataResult.getResults().stream()
|
||||
.filter(r -> r.get(ReferenceEntity.KnownFields.Label).toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
|
||||
.collect(Collectors.toList()));
|
||||
|
|
@ -310,28 +309,30 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
boolean isValid = true;
|
||||
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
|
||||
if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue;
|
||||
|
||||
boolean getFirst = field.getResponsePath().endsWith(".first()");
|
||||
String responsePath = new String(field.getResponsePath());
|
||||
if (getFirst) responsePath = responsePath.substring(0, responsePath.length() - ".first()".length());
|
||||
try {
|
||||
if (field.getResponsePath().contains("@{{")){
|
||||
if (responsePath.contains("@{{")){
|
||||
String rePattern = "@\\{\\{(.*?)}}";
|
||||
Pattern p = Pattern.compile(rePattern);
|
||||
Matcher m = p.matcher(field.getResponsePath());
|
||||
String value = field.getResponsePath();
|
||||
Matcher m = p.matcher(responsePath);
|
||||
String value = responsePath;
|
||||
while (m.find()) {
|
||||
if (m.groupCount() < 1) continue;
|
||||
Object partValue = JsonPath.parse(resultItem).read(m.group(1));
|
||||
String normalizedValue = normalizeJsonValue(partValue);
|
||||
String normalizedValue = normalizeJsonValue(partValue, getFirst);
|
||||
|
||||
if (normalizedValue != null) value = value.replace("@{{" + m.group(1) + "}}", normalizedValue);
|
||||
else value = value.replace("@{{" + m.group(1) + "}}","");
|
||||
}
|
||||
map.put(field.getCode(), normalizeJsonValue(value));
|
||||
map.put(field.getCode(), normalizeJsonValue(value, getFirst));
|
||||
} else {
|
||||
Object value = JsonPath.parse(resultItem).read(field.getResponsePath());
|
||||
map.put(field.getCode(), normalizeJsonValue(value));
|
||||
Object value = JsonPath.parse(resultItem).read(responsePath);
|
||||
map.put(field.getCode(), normalizeJsonValue(value, getFirst));
|
||||
}
|
||||
}catch (PathNotFoundException e){
|
||||
logger.debug("Json Path Error: " + e.getMessage() + " on source " + jsonHandlingService.toJsonSafe(resultItem));
|
||||
logger.debug("Json Path Error: " + e.getMessage() + " on source " + this.jsonHandlingService.toJsonSafe(resultItem));
|
||||
if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) {
|
||||
isValid = false;
|
||||
break;
|
||||
|
|
@ -339,7 +340,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
}
|
||||
}
|
||||
if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){
|
||||
logger.warn("Invalid reference on source " + jsonHandlingService.toJsonSafe(resultItem));
|
||||
logger.warn("Invalid reference on source " + this.jsonHandlingService.toJsonSafe(resultItem));
|
||||
}
|
||||
if (isValid) parsedData.add(map);
|
||||
}
|
||||
|
|
@ -348,29 +349,13 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
|||
return result;
|
||||
}
|
||||
|
||||
private static String normalizeJsonValue(Object value) {
|
||||
private static String normalizeJsonValue(Object value, boolean getfirst) {
|
||||
if (value instanceof JSONArray jsonArray) {
|
||||
|
||||
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
|
||||
if (getfirst) {
|
||||
return jsonArray.getFirst().toString();
|
||||
} else {
|
||||
for (Object o : jsonArray) {
|
||||
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
|
||||
try {
|
||||
return String.valueOf(((Map<?, ?>) o).get("content"));
|
||||
return value.toString();
|
||||
}
|
||||
catch (ClassCastException e){
|
||||
if(((Map<?, ?>) o).get("content") instanceof Integer) {
|
||||
return String.valueOf(((Map<?, ?>) o).get("content"));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Map) {
|
||||
String key = ((Map<?, ?>)value).containsKey("$") ? "$" : "content";
|
||||
return String.valueOf(((Map<?, ?>)value).get(key));
|
||||
}
|
||||
return value != null ? value.toString() : null;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue