package eu.eudat.service.remotefetcher; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.ReferenceTypeSourceType; import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.convention.ConventionService; import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.models.ExternalDataResult; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import gr.cite.tools.exception.MyApplicationException; import net.minidev.json.JSONArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.*; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; import java.io.File; import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; @Service public class RemoteFetcherServiceImpl implements RemoteFetcherService { private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class); private WebClient webClient; private final ExternalUrlConfigProvider externalUrlConfigProvider; private final ConventionService conventionService; private final XmlHandlingService xmlHandlingService; @Autowired public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) { this.externalUrlConfigProvider = externalUrlConfigProvider; this.conventionService = conventionService; this.xmlHandlingService = xmlHandlingService; } private WebClient getWebClient(){ if (this.webClient == null) { this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> { clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here??? } ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build(); } return webClient; } @Override public ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) { List apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList(); if (!this.conventionService.isNullOrEmpty(key)){ apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList(); } if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult(); apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal)); return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria); } @Override public Integer countExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key) { return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size(); } private ExternalDataResult queryExternalData(List sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) { ExternalDataResult results = new ExternalDataResult(); if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult(); for (SourceBaseConfiguration source : sources) { if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) { try { SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source; this.applyFunderToQuery(apiSource, externalReferenceCriteria); String auth = null; if (apiSource.getAuth() != null) { auth = this.buildAuthentication(apiSource.getAuth()); } results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth)); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); } } else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) { SourceStaticOptionConfiguration staticSource = (SourceStaticOptionConfiguration)source; results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO: } } return results; } private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) { HttpMethod method; switch (authenticationConfiguration.getAuthMethod()) { case GET -> method = HttpMethod.GET; case POST -> method =HttpMethod.POST; default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod()); } Map response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl()) .contentType(MediaType.APPLICATION_JSON) .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference>() { })).block(); if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed"); return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null); } private void applyFunderToQuery(SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) { //TODO new reference logic // if (apiSource.getFunderQuery() != null) { // if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) { // apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery())); // } // else { // apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", "")); // } // } } private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List queryConfigs) { String finalQuery = ""; if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery; QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition())) .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null); if (queryConfig != null) { if (queryConfig.getSeparator() != null) { String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator()); finalQuery = queryConfig.getValue(); for (int i = 0; i < likes.length; i++) { finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]); } } else { finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike()); } } return finalQuery; } protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List queries) { if (!this.conventionService.isNullOrEmpty(path)) return path; String completedPath = path; if (externalReferenceCriteria.getLike() != null) { if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) { completedPath = completedPath.replaceAll("\\{like}", "*"); completedPath = completedPath.replaceAll("\\{query}", "*"); } else { if (completedPath.contains("{query}")) { completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries)); } else { completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike()); } } } else { completedPath = completedPath.replace("{like}", ""); } if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) { String[] founderParts = externalReferenceCriteria.getFunderId().split(":"); String funderPrefix = founderParts.length > 0 ? founderParts[0] : ""; String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", ""); if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') { funderId = externalReferenceCriteria.getFunderId(); } completedPath = completedPath.replace("{funderId}", funderId); } else if(completedPath.contains("{funderId}")){ logger.warn("FunderId is null."); completedPath = completedPath.replace("{funderId}", " "); } if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage()); else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage); else completedPath = completedPath.replace("{page}", "1"); completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60"); completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : ""); completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : ""); return completedPath; } private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception { String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries()); String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries()); ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth); if(externalDataResult != null) { if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) { externalDataResult.setResults(externalDataResult.getResults().stream() .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) .collect(Collectors.toList())); } externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList()); if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult; Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); if (externalDataResult.getResults().size() > maxResults) throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query"); return externalDataResult; } else { return new ExternalDataResult(); } } protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) { try { JsonNode jsonBody = new ObjectMapper().readTree(requestBody); HttpMethod method; switch (apiSource.getHttpMethod()) { case GET -> method = HttpMethod.GET; case POST -> method =HttpMethod.POST; default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod()); } ResponseEntity response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> { if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) { httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType()))); httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType())); } if (auth != null) { httpHeaders.set("Authorization", auth); } }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null; //do here all the parsing List responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null); String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : ""; if (responseContentType.contains("json") ) { DocumentContext jsonContext = JsonPath.parse(response.getBody()); return this.parseData(jsonContext, apiSource.getResults()); } else { throw new MyApplicationException("Unsupported response type" + responseContentType); } } catch (Exception exception) { logger.error(exception.getMessage(), exception); } return null; } private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration resultsConfigurationEntity) { ExternalDataResult result = new ExternalDataResult(); if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult(); List> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath()); result.setRawData(rawData); if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult(); List> parsedData = new ArrayList<>(); for (Map stringObjectMap: rawData){ Map map = new HashMap<>(); for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){ String pathValue = field.getResponsePath(); if (!pathValue.contains(".")){ if (stringObjectMap.containsKey(pathValue)) { map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); } }else { if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ String value = null; Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); if (fieldObj != null){ if (fieldObj instanceof Map){ Object o = ((Map) fieldObj).get(pathValue.split("\\.")[1]); if(o instanceof String){ value = (String)o; } else if(o instanceof Integer){ value = String.valueOf(o); } } else if (fieldObj instanceof List) { Object o = ((List>) fieldObj).get(0).get(pathValue.split("\\.")[1]); if(o instanceof String){ value = (String)o; } else if(o instanceof Integer){ value = String.valueOf(o); } } } if (value != null){ map.put(field.getCode(), value); } } } } parsedData.add(map); } result.setResults(parsedData); return result; } private static String normalizeValue(Object value) { if (value instanceof JSONArray jsonArray) { if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) { return jsonArray.getFirst().toString(); } else { for (Object o : jsonArray) { if ((o instanceof Map) && ((Map) o).containsKey("content")) { try { return ((Map) o).get("content"); } catch (ClassCastException e){ if(((Map) o).get("content") instanceof Integer) { return String.valueOf(((Map) o).get("content")); } return null; } } } } } else if (value instanceof Map) { String key = ((Map)value).containsKey("$") ? "$" : "content"; return ((Map)value).get(key); } return value != null ? value.toString() : null; } private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) { List> internalResults; try { String filePath = Paths.get(path).toUri().toURL().toString(); ObjectMapper mapper = new ObjectMapper(); internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); return new ExternalDataResult(searchListMap(internalResults, query)); } catch (Exception e) { logger.error(e.getMessage(), e); return new ExternalDataResult(); } } private List> searchListMap(List> internalResults, String query) { List> list = new LinkedList<>(); for (Map map : internalResults) { if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) { list.add(map); } if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) { list.add(map); } } return list; } private String parseBodyString(String bodyString) { String finalBodyString = bodyString; if (bodyString.contains("{env:")) { int index = bodyString.indexOf("{env: "); while (index >= 0) { int endIndex = bodyString.indexOf("}", index + 6); String envName = bodyString.substring(index + 6, endIndex); finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName)); index = bodyString.indexOf("{env: ", index + 6); } } return finalBodyString; } }