This commit is contained in:
apapachristou 2020-10-05 11:34:13 +03:00
commit dfe4c38b40
45 changed files with 2596 additions and 1113 deletions

View File

@ -16,7 +16,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.Cacheable;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller; import javax.xml.bind.Unmarshaller;
@ -250,21 +257,32 @@ public class RemoteFetcher {
private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) { private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) {
try { try {
RestTemplate restTemplate = new RestTemplate();
URL url = new URL(urlString.replace(" ", "%20")); HttpHeaders headers = new HttpHeaders();
HttpEntity<String> entity;
ResponseEntity response;
/*
URL url = new URL(urlString.replaceAll(" ", "%20"));
HttpURLConnection con = (HttpURLConnection) url.openConnection(); HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET"); con.setRequestMethod("GET");
*/
if (contentType != null && !contentType.isEmpty()) { if (contentType != null && !contentType.isEmpty()) {
con.setRequestProperty("Accept", contentType); headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
} }
int responseCode = con.getResponseCode(); entity = new HttpEntity<>("parameters", headers);
if (responseCode == HttpURLConnection.HTTP_OK) { // success
if (contentType.contains("json")) {
response = restTemplate.exchange(urlString, HttpMethod.GET, entity, Object.class);
} else {
response = restTemplate.exchange(urlString, HttpMethod.GET, entity, String.class);
}
if (response.getStatusCode() == HttpStatus.OK) { // success
//do here all the parsing //do here all the parsing
Results results = new Results(); Results results = new Results();
if (con.getHeaderField("Content-Type").contains("json")) { if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
DocumentContext jsonContext = JsonPath.parse(con.getInputStream()); DocumentContext jsonContext = JsonPath.parse(response.getBody());
if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) { if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) {
results = new Results(jsonContext.read(jsonDataPath.getPath() results = new Results(jsonContext.read(jsonDataPath.getPath()
@ -365,11 +383,12 @@ public class RemoteFetcher {
results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) results.results = results.results.stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
else if (con.getHeaderField("Content-Type").contains("xml")) { else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
Class<?> aClass = Class.forName(jsonDataPath.getParseClass()); Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
JAXBContext jaxbContext = JAXBContext.newInstance(aClass); JAXBContext jaxbContext = JAXBContext.newInstance(aClass);
Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
Object data = unmarshaller.unmarshal(con.getInputStream()); StringReader stringReader = new StringReader(response.getBody().toString());
Object data = unmarshaller.unmarshal(stringReader);
Method reader = null; Method reader = null;
if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) { if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
reader = new PropertyDescriptor(jsonDataPath.getParseField(), aClass).getReadMethod(); reader = new PropertyDescriptor(jsonDataPath.getParseField(), aClass).getReadMethod();
@ -419,13 +438,7 @@ public class RemoteFetcher {
return results; return results;
} }
} catch (MalformedURLException e1) { } catch (Exception exception) {
logger.error(e1.getMessage(), e1);
} //maybe print smth...
catch (IOException e2) {
logger.error(e2.getMessage(), e2);
} //maybe print smth...
catch (Exception exception) {
logger.error(exception.getMessage(), exception); logger.error(exception.getMessage(), exception);
} //maybe print smth... } //maybe print smth...
finally { finally {

Binary file not shown.

After

Width:  |  Height:  |  Size: 124 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 55 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 76 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 114 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 164 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 67 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 232 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 105 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 62 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 86 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 104 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 72 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 60 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 107 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 55 KiB

File diff suppressed because it is too large Load Diff