external fetcher fixes add first logic
This commit is contained in:
parent
81156955dc
commit
1474eda8b9
|
@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.jayway.jsonpath.DocumentContext;
|
import com.jayway.jsonpath.DocumentContext;
|
||||||
import com.jayway.jsonpath.JsonPath;
|
import com.jayway.jsonpath.JsonPath;
|
||||||
import com.jayway.jsonpath.PathNotFoundException;
|
import com.jayway.jsonpath.PathNotFoundException;
|
||||||
|
import gr.cite.tools.exception.MyApplicationException;
|
||||||
|
import net.minidev.json.JSONArray;
|
||||||
import org.opencdmp.commons.JsonHandlingService;
|
import org.opencdmp.commons.JsonHandlingService;
|
||||||
import org.opencdmp.commons.enums.ExternalFetcherSourceType;
|
import org.opencdmp.commons.enums.ExternalFetcherSourceType;
|
||||||
import org.opencdmp.commons.types.externalfetcher.StaticOptionEntity;
|
import org.opencdmp.commons.types.externalfetcher.StaticOptionEntity;
|
||||||
|
@ -13,25 +15,22 @@ import org.opencdmp.data.ReferenceEntity;
|
||||||
import org.opencdmp.model.Reference;
|
import org.opencdmp.model.Reference;
|
||||||
import org.opencdmp.model.referencedefinition.Field;
|
import org.opencdmp.model.referencedefinition.Field;
|
||||||
import org.opencdmp.service.externalfetcher.config.entities.*;
|
import org.opencdmp.service.externalfetcher.config.entities.*;
|
||||||
import org.opencdmp.service.externalfetcher.models.ExternalDataResult;
|
|
||||||
import org.opencdmp.service.externalfetcher.criteria.ExternalReferenceCriteria;
|
import org.opencdmp.service.externalfetcher.criteria.ExternalReferenceCriteria;
|
||||||
import gr.cite.tools.exception.MyApplicationException;
|
import org.opencdmp.service.externalfetcher.models.ExternalDataResult;
|
||||||
import io.netty.handler.ssl.SslContext;
|
|
||||||
import io.netty.handler.ssl.SslContextBuilder;
|
|
||||||
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
|
|
||||||
import net.minidev.json.JSONArray;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.core.ParameterizedTypeReference;
|
import org.springframework.core.ParameterizedTypeReference;
|
||||||
import org.springframework.http.*;
|
import org.springframework.http.HttpMethod;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||||
import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.web.reactive.function.client.WebClient;
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
import reactor.netty.http.client.HttpClient;
|
import reactor.netty.http.client.HttpClient;
|
||||||
|
|
||||||
import javax.net.ssl.SSLException;
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
@ -58,7 +57,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
}
|
}
|
||||||
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
|
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
|
||||||
}
|
}
|
||||||
return webClient;
|
return this.webClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -78,7 +77,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
|
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
|
||||||
return getExternalData(sources, externalReferenceCriteria, key).getResults().size();
|
return this.getExternalData(sources, externalReferenceCriteria, key).getResults().size();
|
||||||
}
|
}
|
||||||
|
|
||||||
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria) {
|
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||||
|
@ -103,7 +102,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
}
|
}
|
||||||
} else if (source.getType() != null && source.getType().equals(ExternalFetcherSourceType.STATIC)) {
|
} else if (source.getType() != null && source.getType().equals(ExternalFetcherSourceType.STATIC)) {
|
||||||
SourceStaticOptionConfiguration<Static> staticSource = (SourceStaticOptionConfiguration)source;
|
SourceStaticOptionConfiguration<Static> staticSource = (SourceStaticOptionConfiguration)source;
|
||||||
results.addAll(queryStaticData(staticSource, externalReferenceCriteria));
|
results.addAll(this.queryStaticData(staticSource, externalReferenceCriteria));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
|
@ -232,12 +231,12 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
|
private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
|
||||||
String replacedPath = replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
|
String replacedPath = this.replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
|
||||||
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
|
String replacedBody = this.replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
|
||||||
|
|
||||||
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
|
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
|
||||||
if(externalDataResult != null) {
|
if(externalDataResult != null) {
|
||||||
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
|
if (apiSource.getFilterType() != null && "local".equals(apiSource.getFilterType()) && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
|
||||||
externalDataResult.setResults(externalDataResult.getResults().stream()
|
externalDataResult.setResults(externalDataResult.getResults().stream()
|
||||||
.filter(r -> r.get(ReferenceEntity.KnownFields.Label).toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
|
.filter(r -> r.get(ReferenceEntity.KnownFields.Label).toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
|
||||||
.collect(Collectors.toList()));
|
.collect(Collectors.toList()));
|
||||||
|
@ -310,28 +309,30 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
boolean isValid = true;
|
boolean isValid = true;
|
||||||
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
|
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
|
||||||
if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue;
|
if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue;
|
||||||
|
boolean getFirst = field.getResponsePath().endsWith(".first()");
|
||||||
|
String responsePath = new String(field.getResponsePath());
|
||||||
|
if (getFirst) responsePath = responsePath.substring(0, responsePath.length() - ".first()".length());
|
||||||
try {
|
try {
|
||||||
if (field.getResponsePath().contains("@{{")){
|
if (responsePath.contains("@{{")){
|
||||||
String rePattern = "@\\{\\{(.*?)}}";
|
String rePattern = "@\\{\\{(.*?)}}";
|
||||||
Pattern p = Pattern.compile(rePattern);
|
Pattern p = Pattern.compile(rePattern);
|
||||||
Matcher m = p.matcher(field.getResponsePath());
|
Matcher m = p.matcher(responsePath);
|
||||||
String value = field.getResponsePath();
|
String value = responsePath;
|
||||||
while (m.find()) {
|
while (m.find()) {
|
||||||
if (m.groupCount() < 1) continue;
|
if (m.groupCount() < 1) continue;
|
||||||
Object partValue = JsonPath.parse(resultItem).read(m.group(1));
|
Object partValue = JsonPath.parse(resultItem).read(m.group(1));
|
||||||
String normalizedValue = normalizeJsonValue(partValue);
|
String normalizedValue = normalizeJsonValue(partValue, getFirst);
|
||||||
|
|
||||||
if (normalizedValue != null) value = value.replace("@{{" + m.group(1) + "}}", normalizedValue);
|
if (normalizedValue != null) value = value.replace("@{{" + m.group(1) + "}}", normalizedValue);
|
||||||
else value = value.replace("@{{" + m.group(1) + "}}","");
|
else value = value.replace("@{{" + m.group(1) + "}}","");
|
||||||
}
|
}
|
||||||
map.put(field.getCode(), normalizeJsonValue(value));
|
map.put(field.getCode(), normalizeJsonValue(value, getFirst));
|
||||||
} else {
|
} else {
|
||||||
Object value = JsonPath.parse(resultItem).read(field.getResponsePath());
|
Object value = JsonPath.parse(resultItem).read(responsePath);
|
||||||
map.put(field.getCode(), normalizeJsonValue(value));
|
map.put(field.getCode(), normalizeJsonValue(value, getFirst));
|
||||||
}
|
}
|
||||||
}catch (PathNotFoundException e){
|
}catch (PathNotFoundException e){
|
||||||
logger.debug("Json Path Error: " + e.getMessage() + " on source " + jsonHandlingService.toJsonSafe(resultItem));
|
logger.debug("Json Path Error: " + e.getMessage() + " on source " + this.jsonHandlingService.toJsonSafe(resultItem));
|
||||||
if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) {
|
if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) {
|
||||||
isValid = false;
|
isValid = false;
|
||||||
break;
|
break;
|
||||||
|
@ -339,7 +340,7 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){
|
if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){
|
||||||
logger.warn("Invalid reference on source " + jsonHandlingService.toJsonSafe(resultItem));
|
logger.warn("Invalid reference on source " + this.jsonHandlingService.toJsonSafe(resultItem));
|
||||||
}
|
}
|
||||||
if (isValid) parsedData.add(map);
|
if (isValid) parsedData.add(map);
|
||||||
}
|
}
|
||||||
|
@ -348,29 +349,13 @@ public class ExternalFetcherServiceImpl implements ExternalFetcherService {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String normalizeJsonValue(Object value) {
|
private static String normalizeJsonValue(Object value, boolean getfirst) {
|
||||||
if (value instanceof JSONArray jsonArray) {
|
if (value instanceof JSONArray jsonArray) {
|
||||||
|
if (getfirst) {
|
||||||
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
|
|
||||||
return jsonArray.getFirst().toString();
|
return jsonArray.getFirst().toString();
|
||||||
} else {
|
} else {
|
||||||
for (Object o : jsonArray) {
|
return value.toString();
|
||||||
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
|
|
||||||
try {
|
|
||||||
return String.valueOf(((Map<?, ?>) o).get("content"));
|
|
||||||
}
|
|
||||||
catch (ClassCastException e){
|
|
||||||
if(((Map<?, ?>) o).get("content") instanceof Integer) {
|
|
||||||
return String.valueOf(((Map<?, ?>) o).get("content"));
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else if (value instanceof Map) {
|
|
||||||
String key = ((Map<?, ?>)value).containsKey("$") ? "$" : "content";
|
|
||||||
return String.valueOf(((Map<?, ?>)value).get(key));
|
|
||||||
}
|
}
|
||||||
return value != null ? value.toString() : null;
|
return value != null ? value.toString() : null;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue