package eu.eudat.logic.proxy.fetching; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import eu.eudat.logic.proxy.config.DataUrlConfiguration; import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.FetchStrategy; import eu.eudat.logic.proxy.config.UrlConfiguration; import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import net.minidev.json.JSONArray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import javax.xml.bind.JAXBContext; import javax.xml.bind.Unmarshaller; import java.beans.PropertyDescriptor; import java.io.*; import java.lang.reflect.Method; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; @Service public class RemoteFetcher { private static final Logger logger = LoggerFactory.getLogger(RemoteFetcher.class); private ConfigLoader configLoader; @Autowired public RemoteFetcher(ConfigLoader configLoader) { this.configLoader = configLoader; } @Cacheable("repositories") public List> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getRepositories().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("grants") public List> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getGrants().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("projects") public List> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("funders") public List> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getFunders().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("organisations") public List> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getOrganisations().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("registries") public List> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getRegistries().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("services") public List> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getServices().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("researchers") public List> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getResearchers().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } /*@Cacheable("tags") public List> getTags(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getTags().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); }*/ @Cacheable("externalDatasets") public List> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getDatasets().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("licenses") public List> getlicenses(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getLicenses().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getLicenses().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getLicenses().getFetchMode(); return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } public Integer findEntries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getValidations().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getValidations().getFetchMode(); List> data = this.getAll(urlConfigs, fetchStrategy, externalUrlCriteria); return data.size(); } private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List> results = new LinkedList<>(); if (urlConfigs == null || urlConfigs.isEmpty()) return results; // throw new NoURLFound("No Repository urls found in configuration"); urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); for (UrlConfiguration urlConfig : urlConfigs) { ifFunderQueryExist(urlConfig, externalUrlCriteria); if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType())); } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); } } return results; } private void ifFunderQueryExist(UrlConfiguration urlConfiguration, ExternalUrlCriteria externalUrlCriteria) { if (urlConfiguration.getFunderQuery() != null) { if (externalUrlCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) { urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery())); } else { urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", "")); } } } private String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria, String firstPage) { String completedPath = path; if (externalUrlCriteria.getLike() != null) { if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror")) && externalUrlCriteria.getLike().equals("")) { completedPath = completedPath.replaceAll("\\{like}", "*"); } else { completedPath = completedPath.replaceAll("\\{like}", externalUrlCriteria.getLike()); } } else { completedPath = completedPath.replace("{like}", ""); } if (externalUrlCriteria.getFunderId() != null) { String funderPrefix = externalUrlCriteria.getFunderId().split(":")[0]; String funderId = externalUrlCriteria.getFunderId().replace(funderPrefix + ":", ""); if (funderId.toCharArray()[0] == ':') { funderId = externalUrlCriteria.getFunderId(); } /* try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); } */ completedPath = completedPath.replace("{funderId}", funderId); } if (externalUrlCriteria.getPage() != null) { completedPath = completedPath.replace("{page}", externalUrlCriteria.getPage()); } else { if (firstPage != null) { completedPath = completedPath.replace("{page}", firstPage); } else { completedPath = completedPath.replace("{page}", "1"); } } if (externalUrlCriteria.getPageSize() != null) { completedPath = completedPath.replace("{pageSize}", externalUrlCriteria.getPageSize()); } else { completedPath = completedPath.replace("{pageSize}", "60"); } if (externalUrlCriteria.getHost() != null) { completedPath = completedPath.replace("{host}", externalUrlCriteria.getHost()); } else { completedPath = completedPath.replace("{host}", ""); } if (externalUrlCriteria.getPath() != null) { completedPath = completedPath.replace("{path}", externalUrlCriteria.getPath()); } else { completedPath = completedPath.replace("{path}", ""); } return completedPath; } private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws HugeResultSet { Set pages = new HashSet<>(); String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage); String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage); Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType); if (fetchStrategy == FetchStrategy.FIRST) return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); if (results != null && results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set for (int i = 2; i <= results.getPagination().get("pages"); i++) pages.add(i); Long maxResults = configLoader.getExternalUrls().getMaxresults(); if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults)) throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); Optional optionalResults = pages.parallelStream() .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType)) .reduce((result1, result2) -> { result1.getResults().addAll(result2.getResults()); return result1; }); Results remainingResults = optionalResults.orElseGet(Results::new); remainingResults.getResults().addAll(results.getResults()); return remainingResults.getResults().stream().peek(x -> x.put("tag", tag)).collect(Collectors.toList()); } private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) { try { RestTemplate restTemplate = new RestTemplate(); HttpHeaders headers = new HttpHeaders(); HttpEntity entity; ResponseEntity response; /* * URL url = new URL(urlString.replaceAll(" ", "%20")); * * HttpURLConnection con = (HttpURLConnection) url.openConnection(); * con.setRequestMethod("GET"); */ if (contentType != null && !contentType.isEmpty()) { headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); headers.setContentType(MediaType.valueOf(contentType)); } JsonNode jsonBody = new ObjectMapper().readTree(requestBody); entity = new HttpEntity<>(jsonBody, headers); response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class); if (response.getStatusCode() == HttpStatus.OK) { // success //do here all the parsing Results results = new Results(); if (response.getHeaders().get("Content-Type").get(0).contains("json")) { DocumentContext jsonContext = JsonPath.parse(response.getBody()); if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) { results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"), new HashMap<>(1, 1)); } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), new HashMap<>(1, 1)); List> fixedResults = results.getResults().stream().map(item -> { for (int i = 0; i < 2; i++) { String id; if (i == 0) { id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", ""); } else { id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", ""); } if (!(item.get(id) instanceof String)) { Object obj = item.get(id); if (obj instanceof JSONArray) { JSONArray jarr = (JSONArray) obj; if (jarr.get(0) instanceof String) { item.put(id, jarr.get(0).toString()); } else { for (int j = 0; j < jarr.size(); j++) { mapToMap(id, (Map)jarr.get(j), item, i == 1); } } } else { if (obj instanceof Map) { mapToMap(id, (Map) obj, item, i == 1); } else if (obj != null){ item.put(id, obj.toString()); } } } } return item; }).collect(Collectors.toList()); results = new Results(fixedResults, new HashMap<>(1, 1)); } else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getPath() + "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"), new HashMap<>(1, 1)); List> multiResults = results.results.stream().map(result -> { ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(); externalUrlCriteria.setPath(result.get("path")); externalUrlCriteria.setHost(result.get("host")); String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage()); return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType); }).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList()); results = new Results(multiResults, new HashMap<>(1, 1)); } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) { List> tempRes = jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]"); List> finalRes = new ArrayList<>(); tempRes.forEach(map -> { Map resMap = new HashMap<>(); map.forEach((key, value) -> { if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) { resMap.put("tags", ((JSONArray) value).toJSONString()); } else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) { resMap.put(key, ((JSONArray) value).toJSONString()); } else { resMap.put(key, (String) value); } }); finalRes.add(resMap); }); results = new Results(finalRes, new HashMap<>(1, 1)); } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) { results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName() + "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"), new HashMap<>(1, 1)); results.getResults().stream().forEach(entry -> { String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", "")); entry.put("name", name); entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")); entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", "")); }); } else { results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), new HashMap<>(1, 1)); } results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) .collect(Collectors.toList()); } else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) { Class aClass = Class.forName(jsonDataPath.getParseClass()); JAXBContext jaxbContext = JAXBContext.newInstance(aClass); Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); StringReader stringReader = new StringReader(response.getBody()); Object data = unmarshaller.unmarshal(stringReader); Method reader = null; if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) { reader = new PropertyDescriptor(jsonDataPath.getParseField(), aClass).getReadMethod(); } ObjectMapper objectMapper = new ObjectMapper(); List> values = new ArrayList<>(); int max = 1; if (reader != null) { Object invokedField = reader.invoke(data); if (invokedField instanceof Collection) { max = ((Collection) invokedField).size(); } } for (int i = 0; i< max; i++) { Object value; if (reader != null) { Object invokedField = reader.invoke(data); if (invokedField instanceof Collection) { value = ((Collection) invokedField).toArray()[i]; } else { value = invokedField; } } else { value = data; } Map map = objectMapper.convertValue(value, Map.class); if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) { Map finalMap = new HashMap<>(); for (Map.Entry entry : map.entrySet()) { if (jsonDataPath.getMergedFields().contains(entry.getKey())) { if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) { finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue()); } else { finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue()); } } else { finalMap.put(entry.getKey(), entry.getValue()); } } values.add(finalMap); } else { values.add(map); } } results = new Results(values, new HashMap<>(1, 1)); } return results; } } catch (Exception exception) { logger.error(exception.getMessage(), exception); } //maybe print smth... finally { } return null; } private List> getAllResultsFromMockUpJson(String path, String query) { List> internalResults; try { String filePath = Paths.get(path).toUri().toURL().toString(); ObjectMapper mapper = new ObjectMapper(); internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); return searchListMap(internalResults, query); } catch (Exception e) { logger.error(e.getMessage(), e); return new LinkedList<>(); } } private List> searchListMap(List> internalResults, String query) { List> list = new LinkedList<>(); for (Map map : internalResults) { if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) { list.add(map); } if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) { list.add(map); } } return list; } private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) { if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host"; return null; } static class Results { List> results; Map pagination; Results() { this.results = new ArrayList<>(); this.pagination = new HashMap<>(); } Results(List> results, Map pagination) { this.results = results; this.pagination = pagination; } List> getResults() { return results; } public void setResults(List> results) { this.results = results; } Map getPagination() { return pagination; } public void setPagination(Map pagination) { this.pagination = pagination; } } private void mapToMap(String key, Map source, Map destination, boolean isTitle) { if (source != null) { String content = source.get("content"); if (isTitle) { String classId = source.get("classid"); if (classId.equals("main title")) { destination.put(key, content); } } else { destination.put(key, content); } } } }