argos/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java

389 lines
21 KiB
Java

package eu.eudat.service.remotefetcher;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.convention.ConventionService;
import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import gr.cite.tools.exception.MyApplicationException;
import net.minidev.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import java.io.File;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class RemoteFetcherServiceImpl implements RemoteFetcherService {
private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
private WebClient webClient;
private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final ConventionService conventionService;
private final XmlHandlingService xmlHandlingService;
@Autowired
public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) {
this.externalUrlConfigProvider = externalUrlConfigProvider;
this.conventionService = conventionService;
this.xmlHandlingService = xmlHandlingService;
}
private WebClient getWebClient(){
if (this.webClient == null) {
this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
}
return webClient;
}
@Override
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
List<SourceBaseConfiguration> apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList();
if (!this.conventionService.isNullOrEmpty(key)){
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList();
}
if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
}
@Override
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
}
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
ExternalDataResult results = new ExternalDataResult();
if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult();
for (SourceBaseConfiguration source : sources) {
if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) {
try {
SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source;
this.applyFunderToQuery(apiSource, externalReferenceCriteria);
String auth = null;
if (apiSource.getAuth() != null) {
auth = this.buildAuthentication(apiSource.getAuth());
}
results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
} else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) {
SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO:
}
}
return results;
}
private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
HttpMethod method;
switch (authenticationConfiguration.getAuthMethod()) {
case GET -> method = HttpMethod.GET;
case POST -> method =HttpMethod.POST;
default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod());
}
Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
.contentType(MediaType.APPLICATION_JSON)
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
})).block();
if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed");
return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
}
private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
//TODO new reference logic
// if (apiSource.getFunderQuery() != null) {
// if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
// }
// else {
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
// }
// }
}
private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
String finalQuery = "";
if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery;
QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition()))
.min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
if (queryConfig != null) {
if (queryConfig.getSeparator() != null) {
String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
finalQuery = queryConfig.getValue();
for (int i = 0; i < likes.length; i++) {
finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]);
}
} else {
finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
}
}
return finalQuery;
}
protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
if (!this.conventionService.isNullOrEmpty(path)) return path;
String completedPath = path;
if (externalReferenceCriteria.getLike() != null) {
if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) {
completedPath = completedPath.replaceAll("\\{like}", "*");
completedPath = completedPath.replaceAll("\\{query}", "*");
} else {
if (completedPath.contains("{query}")) {
completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries));
} else {
completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
}
}
} else {
completedPath = completedPath.replace("{like}", "");
}
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) {
String[] founderParts = externalReferenceCriteria.getFunderId().split(":");
String funderPrefix = founderParts.length > 0 ? founderParts[0] : "";
String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') {
funderId = externalReferenceCriteria.getFunderId();
}
completedPath = completedPath.replace("{funderId}", funderId);
} else if(completedPath.contains("{funderId}")){
logger.warn("FunderId is null.");
completedPath = completedPath.replace("{funderId}", " ");
}
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage);
else completedPath = completedPath.replace("{page}", "1");
completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60");
completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : "");
completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : "");
return completedPath;
}
private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
if(externalDataResult != null) {
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
externalDataResult.setResults(externalDataResult.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList());
if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
if (externalDataResult.getResults().size() > maxResults)
throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
return externalDataResult;
}
else {
return new ExternalDataResult();
}
}
protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) {
try {
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
HttpMethod method;
switch (apiSource.getHttpMethod()) {
case GET -> method = HttpMethod.GET;
case POST -> method =HttpMethod.POST;
default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod());
}
ResponseEntity<String> response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> {
if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) {
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType())));
httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType()));
}
if (auth != null) {
httpHeaders.set("Authorization", auth);
}
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null;
//do here all the parsing
List<String> responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null);
String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : "";
if (responseContentType.contains("json") ) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
return this.parseData(jsonContext, apiSource.getResults());
} else {
throw new MyApplicationException("Unsupported response type" + responseContentType);
}
} catch (Exception exception) {
logger.error(exception.getMessage(), exception);
}
return null;
}
private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
ExternalDataResult result = new ExternalDataResult();
if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
result.setRawData(rawData);
if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
List<Map<String, String>> parsedData = new ArrayList<>();
for (Map<String, Object> stringObjectMap: rawData){
Map<String, String> map = new HashMap<>();
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){
String pathValue = field.getResponsePath();
if (!pathValue.contains(".")){
if (stringObjectMap.containsKey(pathValue)) {
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
}
}else {
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
String value = null;
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
if (fieldObj != null){
if (fieldObj instanceof Map){
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
} else if (fieldObj instanceof List) {
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
}
}
if (value != null){
map.put(field.getCode(), value);
}
}
}
}
parsedData.add(map);
}
result.setResults(parsedData);
return result;
}
private static String normalizeValue(Object value) {
if (value instanceof JSONArray jsonArray) {
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
return jsonArray.getFirst().toString();
} else {
for (Object o : jsonArray) {
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
try {
return ((Map<String, String>) o).get("content");
}
catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) {
return String.valueOf(((Map<?, ?>) o).get("content"));
}
return null;
}
}
}
}
} else if (value instanceof Map) {
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
return ((Map<String, String>)value).get(key);
}
return value != null ? value.toString() : null;
}
private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
List<Map<String, String>> internalResults;
try {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
return new ExternalDataResult(searchListMap(internalResults, query));
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new ExternalDataResult();
}
}
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
List<Map<String, String>> list = new LinkedList<>();
for (Map<String, String> map : internalResults)
{
if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
}
return list;
}
private String parseBodyString(String bodyString) {
String finalBodyString = bodyString;
if (bodyString.contains("{env:")) {
int index = bodyString.indexOf("{env: ");
while (index >= 0) {
int endIndex = bodyString.indexOf("}", index + 6);
String envName = bodyString.substring(index + 6, endIndex);
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
index = bodyString.indexOf("{env: ", index + 6);
}
}
return finalBodyString;
}
}