2024-01-03 17:22:24 +01:00
package eu.eudat.service.reference.external ;
2017-11-21 17:29:16 +01:00
2019-09-05 16:44:25 +02:00
import com.fasterxml.jackson.core.type.TypeReference ;
2020-11-06 17:45:20 +01:00
import com.fasterxml.jackson.databind.JsonNode ;
2019-09-05 16:44:25 +02:00
import com.fasterxml.jackson.databind.ObjectMapper ;
2017-11-21 17:29:16 +01:00
import com.jayway.jsonpath.DocumentContext ;
import com.jayway.jsonpath.JsonPath ;
2023-10-26 13:38:18 +02:00
import eu.eudat.commons.enums.ReferenceType ;
2024-01-03 17:22:24 +01:00
import eu.eudat.commons.exceptions.HugeResultSetException ;
import eu.eudat.service.reference.external.config.* ;
import eu.eudat.service.reference.external.config.entities.GenericUrls ;
import eu.eudat.service.reference.external.models.ExternalRefernceResult ;
import eu.eudat.service.reference.external.criteria.ExternalReferenceCriteria ;
import eu.eudat.service.reference.external.criteria.FetchStrategy ;
import eu.eudat.service.storage.StorageFileService ;
import gr.cite.tools.exception.MyNotFoundException ;
2023-10-06 17:07:58 +02:00
import jakarta.xml.bind.JAXBContext ;
import jakarta.xml.bind.Unmarshaller ;
2020-01-16 16:46:24 +01:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2018-02-16 11:34:02 +01:00
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.cache.annotation.Cacheable ;
2023-10-06 17:07:58 +02:00
import org.springframework.core.ParameterizedTypeReference ;
2021-10-13 16:48:46 +02:00
import org.springframework.http.* ;
2023-10-06 17:07:58 +02:00
import org.springframework.http.client.reactive.ReactorClientHttpConnector ;
import org.springframework.http.codec.json.Jackson2JsonDecoder ;
2018-02-16 11:34:02 +01:00
import org.springframework.stereotype.Service ;
2020-10-05 10:26:35 +02:00
import org.springframework.web.client.RestTemplate ;
2023-10-06 17:07:58 +02:00
import org.springframework.web.reactive.function.client.WebClient ;
import reactor.netty.http.client.HttpClient ;
2018-02-16 11:34:02 +01:00
2024-01-03 17:22:24 +01:00
import java.io.ByteArrayInputStream ;
2021-10-13 16:48:46 +02:00
import java.io.File ;
import java.io.StringReader ;
2020-02-10 17:24:15 +01:00
import java.lang.reflect.Method ;
2019-11-08 14:53:46 +01:00
import java.nio.file.Paths ;
2018-02-16 11:34:02 +01:00
import java.util.* ;
2018-05-28 11:50:42 +02:00
import java.util.stream.Collectors ;
2017-11-21 17:29:16 +01:00
2017-11-22 09:57:51 +01:00
@Service
2017-11-21 17:29:16 +01:00
public class RemoteFetcher {
2020-01-16 16:46:24 +01:00
private static final Logger logger = LoggerFactory . getLogger ( RemoteFetcher . class ) ;
2018-01-17 16:06:35 +01:00
2023-10-06 17:07:58 +02:00
private final WebClient client ;
2024-01-03 17:22:24 +01:00
private final ExternalUrlConfigProvider externalUrlConfigProvider ;
2018-03-28 15:24:47 +02:00
@Autowired
2024-01-03 17:22:24 +01:00
public RemoteFetcher ( ExternalUrlConfigProvider externalUrlConfigProvider ) {
this . externalUrlConfigProvider = externalUrlConfigProvider ;
this . client = WebClient . builder ( ) . codecs ( clientCodecConfigurer - > {
2023-10-06 17:07:58 +02:00
clientCodecConfigurer . defaultCodecs ( ) . jackson2JsonDecoder ( new Jackson2JsonDecoder ( new ObjectMapper ( ) , MediaType . APPLICATION_JSON ) ) ;
clientCodecConfigurer . defaultCodecs ( ) . maxInMemorySize ( 2 * ( ( int ) Math . pow ( 1024 , 3 ) ) ) ; //GK: Why here???
}
) . clientConnector ( new ReactorClientHttpConnector ( HttpClient . create ( ) . followRedirect ( true ) ) ) . build ( ) ;
2018-03-28 15:24:47 +02:00
}
2018-01-17 16:06:35 +01:00
2024-01-03 17:22:24 +01:00
public List < Map < String , String > > get ( ReferenceType referenceType , ExternalReferenceCriteria externalReferenceCriteria , String key ) throws MyNotFoundException , HugeResultSetException {
2023-10-18 17:05:39 +02:00
FetchStrategy fetchStrategy = null ;
2023-10-31 16:57:41 +01:00
GenericUrls exGenericUrls = this . getExternalUrls ( referenceType ) ;
2023-10-19 16:56:53 +02:00
List < UrlConfiguration > urlConfigs = key ! = null & & ! key . isEmpty ( ) ? exGenericUrls . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: exGenericUrls . getUrls ( ) ;
2023-10-18 17:05:39 +02:00
2024-01-03 17:22:24 +01:00
List < Map < String , String > > results = getAll ( urlConfigs , fetchStrategy , externalReferenceCriteria ) ;
2023-10-31 16:57:41 +01:00
for ( Map < String , String > result : results ) {
result . put ( " referenceType " , referenceType . name ( ) ) ;
}
return results ;
2023-10-18 17:05:39 +02:00
}
2023-10-31 16:57:41 +01:00
public GenericUrls getExternalUrls ( ReferenceType referenceType ) {
switch ( referenceType ) {
2024-01-03 17:22:24 +01:00
case Taxonomies : return this . externalUrlConfigProvider . getExternalUrls ( ) . getTaxonomies ( ) ;
case Licenses : return this . externalUrlConfigProvider . getExternalUrls ( ) . getLicenses ( ) ;
case Publications : return this . externalUrlConfigProvider . getExternalUrls ( ) . getPublications ( ) ;
case Journals : return this . externalUrlConfigProvider . getExternalUrls ( ) . getJournals ( ) ;
case PubRepositories : return this . externalUrlConfigProvider . getExternalUrls ( ) . getPubRepositories ( ) ;
case DataRepositories : return this . externalUrlConfigProvider . getExternalUrls ( ) . getRepositories ( ) ;
case Registries : return this . externalUrlConfigProvider . getExternalUrls ( ) . getRegistries ( ) ;
case Services : return this . externalUrlConfigProvider . getExternalUrls ( ) . getServices ( ) ;
case Grants : return this . externalUrlConfigProvider . getExternalUrls ( ) . getGrants ( ) ;
case Organizations : return this . externalUrlConfigProvider . getExternalUrls ( ) . getOrganisations ( ) ;
case Datasets : return this . externalUrlConfigProvider . getExternalUrls ( ) . getDatasets ( ) ;
case Funder : return this . externalUrlConfigProvider . getExternalUrls ( ) . getFunders ( ) ;
case Project : return this . externalUrlConfigProvider . getExternalUrls ( ) . getProjects ( ) ;
case Researcher : return this . externalUrlConfigProvider . getExternalUrls ( ) . getResearchers ( ) ;
2023-10-31 16:57:41 +01:00
default : throw new IllegalArgumentException ( " Type not found " + referenceType ) ;
2023-10-19 16:56:53 +02:00
}
}
2020-11-05 17:20:37 +01:00
2024-01-03 17:22:24 +01:00
public Integer findEntries ( ExternalReferenceCriteria externalReferenceCriteria , String key ) throws MyNotFoundException , HugeResultSetException {
2020-11-05 17:20:37 +01:00
List < UrlConfiguration > urlConfigs =
2024-01-03 17:22:24 +01:00
key ! = null & & ! key . isEmpty ( ) ? this . externalUrlConfigProvider . getExternalUrls ( ) . getValidations ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: this . externalUrlConfigProvider . getExternalUrls ( ) . getValidations ( ) . getUrls ( ) ;
FetchStrategy fetchStrategy = this . externalUrlConfigProvider . getExternalUrls ( ) . getValidations ( ) . getFetchMode ( ) ;
List < Map < String , String > > data = this . getAll ( urlConfigs , fetchStrategy , externalReferenceCriteria ) ;
2020-11-05 17:20:37 +01:00
return data . size ( ) ;
}
2020-06-26 10:46:18 +02:00
2024-01-03 17:22:24 +01:00
public List < Map < String , String > > getExternalGeneric ( ExternalReferenceCriteria externalReferenceCriteria , GenericUrls genericUrls ) {
2021-12-10 14:37:39 +01:00
List < UrlConfiguration > urlConfigurations = genericUrls . getUrls ( ) ;
FetchStrategy fetchStrategy = genericUrls . getFetchMode ( ) ;
2024-01-03 17:22:24 +01:00
return getAll ( urlConfigurations , fetchStrategy , externalReferenceCriteria ) ;
2021-12-10 14:37:39 +01:00
}
2024-01-03 17:22:24 +01:00
public List < Map < String , Object > > getExternalGenericWithData ( ExternalReferenceCriteria externalReferenceCriteria , GenericUrls genericUrls ) {
2023-07-14 12:47:01 +02:00
List < UrlConfiguration > urlConfigurations = genericUrls . getUrls ( ) ;
2024-01-03 17:22:24 +01:00
return getAllWithData ( urlConfigurations , externalReferenceCriteria ) ;
2023-07-14 12:47:01 +02:00
}
2018-01-17 16:06:35 +01:00
2024-01-03 17:22:24 +01:00
private List < Map < String , String > > getAll ( List < UrlConfiguration > urlConfigs , FetchStrategy fetchStrategy , ExternalReferenceCriteria externalReferenceCriteria ) {
2018-01-17 16:06:35 +01:00
2019-12-12 10:26:19 +01:00
List < Map < String , String > > results = new LinkedList < > ( ) ;
2021-10-13 16:48:46 +02:00
if ( urlConfigs = = null | | urlConfigs . isEmpty ( ) ) {
return results ;
}
2024-01-03 17:22:24 +01:00
// throw new MyNotFoundException("No Repository urls found in configuration");
2018-01-17 16:06:35 +01:00
2019-10-03 13:06:44 +02:00
urlConfigs . sort ( Comparator . comparing ( UrlConfiguration : : getOrdinal ) ) ;
2021-10-13 16:48:46 +02:00
urlConfigs . forEach ( urlConfiguration - > {
2024-01-03 17:22:24 +01:00
ifFunderQueryExist ( urlConfiguration , externalReferenceCriteria ) ;
2021-10-13 16:48:46 +02:00
if ( urlConfiguration . getType ( ) = = null | | urlConfiguration . getType ( ) . equals ( " External " ) ) {
try {
2023-10-06 17:07:58 +02:00
String auth = null ;
if ( urlConfiguration . getAuth ( ) ! = null ) {
auth = this . getAuthentication ( urlConfiguration . getAuth ( ) ) ;
}
2024-01-03 17:22:24 +01:00
results . addAll ( getAllResultsFromUrl ( urlConfiguration . getUrl ( ) , fetchStrategy , urlConfiguration . getData ( ) , urlConfiguration . getPaginationPath ( ) , externalReferenceCriteria , urlConfiguration . getLabel ( ) , urlConfiguration . getKey ( ) , urlConfiguration . getContentType ( ) , urlConfiguration . getFirstpage ( ) , urlConfiguration . getRequestBody ( ) , urlConfiguration . getRequestType ( ) , urlConfiguration . getFilterType ( ) , urlConfiguration . getQueries ( ) , auth ) ) ;
2021-10-13 16:48:46 +02:00
} catch ( Exception e ) {
logger . error ( e . getLocalizedMessage ( ) , e ) ;
}
} else if ( urlConfiguration . getType ( ) ! = null & & urlConfiguration . getType ( ) . equals ( " Internal " ) ) {
2024-01-03 17:22:24 +01:00
results . addAll ( getAllResultsFromMockUpJson ( urlConfiguration . getUrl ( ) , externalReferenceCriteria . getLike ( ) ) ) ;
2021-10-13 16:48:46 +02:00
}
} ) ;
/ * for ( UrlConfiguration urlConfig : urlConfigs ) {
2019-10-03 13:06:44 +02:00
ifFunderQueryExist ( urlConfig , externalUrlCriteria ) ;
2019-09-05 16:44:25 +02:00
if ( urlConfig . getType ( ) = = null | | urlConfig . getType ( ) . equals ( " External " ) ) {
2020-11-06 17:45:20 +01:00
results . addAll ( getAllResultsFromUrl ( urlConfig . getUrl ( ) , fetchStrategy , urlConfig . getData ( ) , urlConfig . getPaginationPath ( ) , externalUrlCriteria , urlConfig . getLabel ( ) , urlConfig . getKey ( ) , urlConfig . getContentType ( ) , urlConfig . getFirstpage ( ) , urlConfig . getRequestBody ( ) , urlConfig . getRequestType ( ) ) ) ;
2019-10-01 12:19:39 +02:00
} else if ( urlConfig . getType ( ) ! = null & & urlConfig . getType ( ) . equals ( " Internal " ) ) {
2019-10-03 13:06:44 +02:00
results . addAll ( getAllResultsFromMockUpJson ( urlConfig . getUrl ( ) , externalUrlCriteria . getLike ( ) ) ) ;
2019-09-05 16:44:25 +02:00
}
2021-10-13 16:48:46 +02:00
} * /
2018-06-29 10:29:43 +02:00
return results ;
2018-01-17 16:06:35 +01:00
}
2023-10-06 17:07:58 +02:00
private String getAuthentication ( AuthenticationConfiguration authenticationConfiguration ) {
HttpMethod method = HttpMethod . valueOf ( authenticationConfiguration . getAuthMethod ( ) ) ;
Map < String , Object > reponse = this . client . method ( method ) . uri ( authenticationConfiguration . getAuthUrl ( ) )
. contentType ( MediaType . APPLICATION_JSON )
. bodyValue ( this . parseBodyString ( authenticationConfiguration . getAuthRequestBody ( ) ) )
. exchangeToMono ( mono - > mono . bodyToMono ( new ParameterizedTypeReference < Map < String , Object > > ( ) {
} ) ) . block ( ) ;
return authenticationConfiguration . getType ( ) + " " + reponse . get ( authenticationConfiguration . getAuthTokenPath ( ) ) ;
}
2024-01-03 17:22:24 +01:00
private List < Map < String , Object > > getAllWithData ( List < UrlConfiguration > urlConfigs , ExternalReferenceCriteria externalReferenceCriteria ) {
2023-07-14 12:47:01 +02:00
List < Map < String , Object > > results = new LinkedList < > ( ) ;
if ( urlConfigs = = null | | urlConfigs . isEmpty ( ) ) {
return results ;
}
urlConfigs . sort ( Comparator . comparing ( UrlConfiguration : : getOrdinal ) ) ;
urlConfigs . forEach ( urlConfiguration - > {
2024-01-03 17:22:24 +01:00
ifFunderQueryExist ( urlConfiguration , externalReferenceCriteria ) ;
2023-07-14 12:47:01 +02:00
if ( urlConfiguration . getType ( ) = = null | | urlConfiguration . getType ( ) . equals ( " External " ) ) {
try {
2024-01-03 17:22:24 +01:00
results . addAll ( getAllResultsFromUrlWithData ( urlConfiguration . getUrl ( ) , urlConfiguration . getData ( ) , externalReferenceCriteria , urlConfiguration . getContentType ( ) , urlConfiguration . getFirstpage ( ) , urlConfiguration . getRequestBody ( ) , urlConfiguration . getRequestType ( ) , urlConfiguration . getQueries ( ) ) ) ;
2023-07-14 12:47:01 +02:00
} catch ( Exception e ) {
logger . error ( e . getLocalizedMessage ( ) , e ) ;
}
}
} ) ;
return results ;
}
2024-01-03 17:22:24 +01:00
private void ifFunderQueryExist ( UrlConfiguration urlConfiguration , ExternalReferenceCriteria externalReferenceCriteria ) {
2019-10-03 13:06:44 +02:00
if ( urlConfiguration . getFunderQuery ( ) ! = null ) {
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getFunderId ( ) ! = null & & ! urlConfiguration . getFunderQuery ( ) . startsWith ( " dmp: " ) ) {
2019-10-03 13:06:44 +02:00
urlConfiguration . setUrl ( urlConfiguration . getUrl ( ) . replace ( " {funderQuery} " , urlConfiguration . getFunderQuery ( ) ) ) ;
}
else {
urlConfiguration . setUrl ( urlConfiguration . getUrl ( ) . replace ( " {funderQuery} " , " " ) ) ;
}
2019-10-01 12:19:39 +02:00
}
2019-10-03 13:06:44 +02:00
}
2018-01-17 16:06:35 +01:00
2024-01-03 17:22:24 +01:00
private String calculateQuery ( ExternalReferenceCriteria externalReferenceCriteria , List < QueryConfig > queryConfigs ) {
2022-05-25 16:37:20 +02:00
String finalQuery = " " ;
2024-01-03 17:22:24 +01:00
QueryConfig queryConfig = queryConfigs . stream ( ) . filter ( queryConfigl - > externalReferenceCriteria . getLike ( ) . matches ( queryConfigl . getCondition ( ) ) )
2022-05-25 17:06:54 +02:00
. min ( ( Comparator . comparing ( QueryConfig : : getOrdinal ) ) ) . orElse ( null ) ;
2022-05-25 16:37:20 +02:00
if ( queryConfig ! = null ) {
if ( queryConfig . getSeparator ( ) ! = null ) {
2024-01-03 17:22:24 +01:00
String [ ] likes = externalReferenceCriteria . getLike ( ) . split ( queryConfig . getSeparator ( ) ) ;
2022-05-25 16:37:20 +02:00
finalQuery = queryConfig . getValue ( ) ;
for ( int i = 0 ; i < likes . length ; i + + ) {
finalQuery = finalQuery . replaceAll ( " \\ {like " + ( i + 1 ) + " } " , likes [ i ] ) ;
}
} else {
2024-01-03 17:22:24 +01:00
finalQuery = queryConfig . getValue ( ) . replaceAll ( " \\ {like} " , externalReferenceCriteria . getLike ( ) ) ;
2022-05-25 16:37:20 +02:00
}
}
return finalQuery ;
}
2024-01-03 17:22:24 +01:00
protected String replaceCriteriaOnUrl ( String path , ExternalReferenceCriteria externalReferenceCriteria , String firstPage , List < QueryConfig > queries ) {
2019-10-03 13:06:44 +02:00
String completedPath = path ;
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getLike ( ) ! = null ) {
if ( ( path . contains ( " openaire " ) | | path . contains ( " orcid " ) | | path . contains ( " ror " ) | | path . contains ( " fairsharing " ) ) & & externalReferenceCriteria . getLike ( ) . equals ( " " ) ) {
2019-10-03 13:06:44 +02:00
completedPath = completedPath . replaceAll ( " \\ {like} " , " * " ) ;
2022-05-25 16:37:20 +02:00
completedPath = completedPath . replaceAll ( " \\ {query} " , " * " ) ;
2020-02-10 17:27:38 +01:00
} else {
2022-05-25 16:37:20 +02:00
if ( completedPath . contains ( " {query} " ) ) {
2024-01-03 17:22:24 +01:00
completedPath = completedPath . replaceAll ( " \\ {query} " , this . calculateQuery ( externalReferenceCriteria , queries ) ) ;
2022-05-25 16:37:20 +02:00
} else {
2024-01-03 17:22:24 +01:00
completedPath = completedPath . replaceAll ( " \\ {like} " , externalReferenceCriteria . getLike ( ) ) ;
2022-05-25 16:37:20 +02:00
}
2020-02-10 17:27:38 +01:00
}
2019-10-01 12:19:39 +02:00
} else {
2019-10-03 13:06:44 +02:00
completedPath = completedPath . replace ( " {like} " , " " ) ;
}
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getFunderId ( ) ! = null ) {
String funderPrefix = externalReferenceCriteria . getFunderId ( ) . split ( " : " ) [ 0 ] ;
String funderId = externalReferenceCriteria . getFunderId ( ) . replace ( funderPrefix + " : " , " " ) ;
2020-05-06 17:02:19 +02:00
if ( funderId . toCharArray ( ) [ 0 ] = = ':' ) {
2024-01-03 17:22:24 +01:00
funderId = externalReferenceCriteria . getFunderId ( ) ;
2020-05-06 17:02:19 +02:00
}
2020-10-05 15:21:42 +02:00
/ *
try { funderId = URLEncoder . encode ( funderId , " UTF-8 " ) ; } catch
( UnsupportedEncodingException e ) { logger . error ( e . getMessage ( ) , e ) ; }
* /
2019-10-03 13:06:44 +02:00
completedPath = completedPath . replace ( " {funderId} " , funderId ) ;
2019-10-01 12:19:39 +02:00
}
2023-06-21 11:17:22 +02:00
else if ( completedPath . contains ( " {funderId} " ) ) {
logger . warn ( " FunderId is null. " ) ;
completedPath = completedPath . replace ( " {funderId} " , " " ) ;
}
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getPage ( ) ! = null ) {
completedPath = completedPath . replace ( " {page} " , externalReferenceCriteria . getPage ( ) ) ;
2019-10-03 13:06:44 +02:00
} else {
2019-10-04 13:33:38 +02:00
if ( firstPage ! = null ) {
completedPath = completedPath . replace ( " {page} " , firstPage ) ;
} else {
completedPath = completedPath . replace ( " {page} " , " 1 " ) ;
}
2019-10-03 13:06:44 +02:00
}
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getPageSize ( ) ! = null ) {
completedPath = completedPath . replace ( " {pageSize} " , externalReferenceCriteria . getPageSize ( ) ) ;
2019-10-03 13:06:44 +02:00
} else {
2019-12-13 12:08:32 +01:00
completedPath = completedPath . replace ( " {pageSize} " , " 60 " ) ;
2019-10-03 13:06:44 +02:00
}
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getHost ( ) ! = null ) {
completedPath = completedPath . replace ( " {host} " , externalReferenceCriteria . getHost ( ) ) ;
2020-02-10 17:24:15 +01:00
} else {
completedPath = completedPath . replace ( " {host} " , " " ) ;
}
2024-01-03 17:22:24 +01:00
if ( externalReferenceCriteria . getPath ( ) ! = null ) {
completedPath = completedPath . replace ( " {path} " , externalReferenceCriteria . getPath ( ) ) ;
2020-02-10 17:24:15 +01:00
} else {
completedPath = completedPath . replace ( " {path} " , " " ) ;
}
2019-10-03 13:06:44 +02:00
return completedPath ;
}
2018-01-17 16:06:35 +01:00
2024-01-03 17:22:24 +01:00
private List < Map < String , String > > getAllResultsFromUrl ( String path , FetchStrategy fetchStrategy , final DataUrlConfiguration jsonDataPath , final String jsonPaginationPath , ExternalReferenceCriteria externalReferenceCriteria , String tag , String key , String contentType , String firstPage , String requestBody , String requestType , String filterType , List < QueryConfig > queries , String auth ) throws Exception {
2019-10-03 13:06:44 +02:00
Set < Integer > pages = new HashSet < > ( ) ;
2024-01-03 17:22:24 +01:00
String replacedPath = replaceCriteriaOnUrl ( path , externalReferenceCriteria , firstPage , queries ) ;
String replacedBody = replaceCriteriaOnUrl ( requestBody , externalReferenceCriteria , firstPage , queries ) ;
2019-10-03 13:06:44 +02:00
2024-01-03 17:22:24 +01:00
ExternalRefernceResult externalRefernceResult = getResultsFromUrl ( replacedPath , jsonDataPath , jsonPaginationPath , contentType , replacedBody , requestType , auth ) ;
if ( externalRefernceResult ! = null ) {
if ( filterType ! = null & & filterType . equals ( " local " ) & & ( externalReferenceCriteria . getLike ( ) ! = null & & ! externalReferenceCriteria . getLike ( ) . isEmpty ( ) ) ) {
externalRefernceResult . setResults ( externalRefernceResult . getResults ( ) . stream ( )
. filter ( r - > r . get ( " name " ) . toLowerCase ( ) . contains ( externalReferenceCriteria . getLike ( ) . toLowerCase ( ) ) )
2023-06-21 11:17:22 +02:00
. collect ( Collectors . toList ( ) ) ) ;
}
if ( fetchStrategy = = FetchStrategy . FIRST )
2024-01-03 17:22:24 +01:00
return externalRefernceResult . getResults ( ) . stream ( ) . peek ( x - > x . put ( " tag " , tag ) ) . peek ( x - > x . put ( " key " , key ) ) . collect ( Collectors . toList ( ) ) ;
2023-06-21 11:17:22 +02:00
2024-01-03 17:22:24 +01:00
if ( externalRefernceResult . getPagination ( ) ! = null & & externalRefernceResult . getPagination ( ) . get ( " pages " ) ! = null ) //if has more pages, add them to the pages set
for ( int i = 2 ; i < = externalRefernceResult . getPagination ( ) . get ( " pages " ) ; i + + )
2023-06-21 11:17:22 +02:00
pages . add ( i ) ;
2024-01-03 17:22:24 +01:00
Long maxResults = this . externalUrlConfigProvider . getExternalUrls ( ) . getMaxresults ( ) ;
if ( ( maxResults > 0 ) & & ( externalRefernceResult . getPagination ( ) . get ( " count " ) > maxResults ) )
throw new HugeResultSetException ( " The submitted search query " + externalReferenceCriteria . getLike ( ) + " is about to return " + externalRefernceResult . getPagination ( ) . get ( " count " ) + " results... Please submit a more detailed search query " ) ;
2023-06-21 11:17:22 +02:00
2024-01-03 17:22:24 +01:00
Optional < ExternalRefernceResult > optionalResults = pages . parallelStream ( )
2023-10-06 17:07:58 +02:00
. map ( page - > getResultsFromUrl ( path + " &page= " + page , jsonDataPath , jsonPaginationPath , contentType , replacedBody , requestType , auth ) )
2023-06-21 11:17:22 +02:00
. filter ( Objects : : nonNull )
. reduce ( ( result1 , result2 ) - > {
result1 . getResults ( ) . addAll ( result2 . getResults ( ) ) ;
return result1 ;
} ) ;
2024-01-03 17:22:24 +01:00
ExternalRefernceResult remainingExternalRefernceResult = optionalResults . orElseGet ( ExternalRefernceResult : : new ) ;
remainingExternalRefernceResult . getResults ( ) . addAll ( externalRefernceResult . getResults ( ) ) ;
2023-06-21 11:17:22 +02:00
2024-01-03 17:22:24 +01:00
return remainingExternalRefernceResult . getResults ( ) . stream ( ) . peek ( x - > x . put ( " tag " , tag ) ) . peek ( x - > x . put ( " key " , key ) ) . collect ( Collectors . toList ( ) ) ;
2023-06-21 11:17:22 +02:00
}
else {
return new LinkedList < > ( ) ;
2022-03-23 15:03:42 +01:00
}
2018-01-17 16:06:35 +01:00
}
2024-01-03 17:22:24 +01:00
private List < Map < String , Object > > getAllResultsFromUrlWithData ( String path , final DataUrlConfiguration jsonDataPath , ExternalReferenceCriteria externalReferenceCriteria , String contentType , String firstPage , String requestBody , String requestType , List < QueryConfig > queries ) {
2023-07-14 12:47:01 +02:00
2024-01-03 17:22:24 +01:00
String replacedPath = replaceCriteriaOnUrl ( path , externalReferenceCriteria , firstPage , queries ) ;
String replacedBody = replaceCriteriaOnUrl ( requestBody , externalReferenceCriteria , firstPage , queries ) ;
2023-07-14 12:47:01 +02:00
try {
RestTemplate restTemplate = new RestTemplate ( ) ;
HttpHeaders headers = new HttpHeaders ( ) ;
HttpEntity < JsonNode > entity ;
ResponseEntity < String > response ;
if ( contentType ! = null & & ! contentType . isEmpty ( ) ) {
headers . setAccept ( Collections . singletonList ( MediaType . valueOf ( contentType ) ) ) ;
headers . setContentType ( MediaType . valueOf ( contentType ) ) ;
}
JsonNode jsonBody = new ObjectMapper ( ) . readTree ( replacedBody ) ;
entity = new HttpEntity < > ( jsonBody , headers ) ;
2023-10-06 17:07:58 +02:00
response = restTemplate . exchange ( replacedPath , HttpMethod . valueOf ( requestType ) , entity , String . class ) ;
2023-07-14 12:47:01 +02:00
if ( response . getStatusCode ( ) = = HttpStatus . OK ) {
if ( response . getHeaders ( ) . get ( " Content-Type " ) . get ( 0 ) . contains ( " json " ) ) {
DocumentContext jsonContext = JsonPath . parse ( response . getBody ( ) ) ;
return jsonContext . read ( jsonDataPath . getPath ( ) ) ;
}
}
}
catch ( Exception exception ) {
logger . error ( exception . getMessage ( ) , exception ) ;
}
return new LinkedList < > ( ) ;
}
2018-01-17 16:06:35 +01:00
2024-01-03 17:22:24 +01:00
protected ExternalRefernceResult getResultsFromUrl ( String urlString , DataUrlConfiguration jsonDataPath , String jsonPaginationPath , String contentType , String requestBody , String requestType , String auth ) {
2018-01-17 16:06:35 +01:00
try {
2023-10-06 17:07:58 +02:00
//RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory());
//HttpHeaders headers = new HttpHeaders();
//HttpEntity<JsonNode> entity;
2020-10-05 15:21:42 +02:00
ResponseEntity < String > response ;
2020-10-05 10:26:35 +02:00
/ *
2020-10-05 15:21:42 +02:00
* URL url = new URL ( urlString . replaceAll ( " " , " %20 " ) ) ;
*
* HttpURLConnection con = ( HttpURLConnection ) url . openConnection ( ) ;
* con . setRequestMethod ( " GET " ) ;
2020-10-05 10:26:35 +02:00
* /
2023-10-06 17:07:58 +02:00
/ * if ( contentType ! = null & & ! contentType . isEmpty ( ) ) {
2020-10-05 10:26:35 +02:00
headers . setAccept ( Collections . singletonList ( MediaType . valueOf ( contentType ) ) ) ;
2020-11-06 17:45:20 +01:00
headers . setContentType ( MediaType . valueOf ( contentType ) ) ;
2020-02-10 17:24:15 +01:00
}
2023-10-06 17:07:58 +02:00
if ( auth ! = null ) {
headers . set ( " Authorization " , auth ) ;
} * /
2020-11-06 17:45:20 +01:00
JsonNode jsonBody = new ObjectMapper ( ) . readTree ( requestBody ) ;
2023-10-06 17:07:58 +02:00
// entity = new HttpEntity<>(jsonBody, headers);
2018-01-17 16:06:35 +01:00
2023-10-06 17:07:58 +02:00
response = this . client . method ( HttpMethod . valueOf ( requestType ) ) . uri ( urlString ) . headers ( httpHeaders - > {
if ( contentType ! = null & & ! contentType . isEmpty ( ) ) {
httpHeaders . setAccept ( Collections . singletonList ( MediaType . valueOf ( contentType ) ) ) ;
httpHeaders . setContentType ( MediaType . valueOf ( contentType ) ) ;
}
if ( auth ! = null ) {
httpHeaders . set ( " Authorization " , auth ) ;
}
} ) . bodyValue ( jsonBody ) . retrieve ( ) . toEntity ( String . class ) . block ( ) ;
//response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class);
2020-10-05 10:26:35 +02:00
if ( response . getStatusCode ( ) = = HttpStatus . OK ) { // success
2018-01-17 16:06:35 +01:00
//do here all the parsing
2024-01-03 17:22:24 +01:00
ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult ( ) ;
2020-10-05 10:26:35 +02:00
if ( response . getHeaders ( ) . get ( " Content-Type " ) . get ( 0 ) . contains ( " json " ) ) {
DocumentContext jsonContext = JsonPath . parse ( response . getBody ( ) ) ;
2020-02-10 17:24:15 +01:00
2022-01-19 11:12:20 +01:00
if ( jsonDataPath . getFieldsUrlConfiguration ( ) . getPath ( ) ! = null ) {
2024-01-03 17:22:24 +01:00
externalRefernceResult = RemoteFetcherUtils . getFromJsonWithRecursiveFetching ( jsonContext , jsonDataPath , this , requestBody , requestType , auth ) ;
2020-09-11 11:07:49 +02:00
} else if ( jsonDataPath . getFieldsUrlConfiguration ( ) . getFirstName ( ) ! = null ) {
2024-01-03 17:22:24 +01:00
externalRefernceResult = RemoteFetcherUtils . getFromJsonWithFirstAndLastName ( jsonContext , jsonDataPath ) ;
2020-02-10 17:24:15 +01:00
} else {
2024-01-03 17:22:24 +01:00
externalRefernceResult = RemoteFetcherUtils . getFromJson ( jsonContext , jsonDataPath ) ;
2020-02-10 17:24:15 +01:00
}
2024-01-03 17:22:24 +01:00
externalRefernceResult . setResults ( externalRefernceResult . getResults ( ) . stream ( ) . map ( e - > e . entrySet ( ) . stream ( ) . collect ( Collectors . toMap ( x - > this . transformKey ( jsonDataPath , x . getKey ( ) ) , Map . Entry : : getValue ) ) )
2021-10-13 16:48:46 +02:00
. collect ( Collectors . toList ( ) ) ) ;
2020-05-29 16:10:18 +02:00
}
2020-10-05 10:26:35 +02:00
else if ( response . getHeaders ( ) . get ( " Content-Type " ) . get ( 0 ) . contains ( " xml " ) ) {
2020-02-10 17:24:15 +01:00
Class < ? > aClass = Class . forName ( jsonDataPath . getParseClass ( ) ) ;
JAXBContext jaxbContext = JAXBContext . newInstance ( aClass ) ;
Unmarshaller unmarshaller = jaxbContext . createUnmarshaller ( ) ;
2020-10-05 15:21:42 +02:00
StringReader stringReader = new StringReader ( response . getBody ( ) ) ;
2020-10-05 10:26:35 +02:00
Object data = unmarshaller . unmarshal ( stringReader ) ;
2020-02-10 17:24:15 +01:00
Method reader = null ;
if ( jsonDataPath . getParseField ( ) ! = null & & ! jsonDataPath . getParseField ( ) . isEmpty ( ) ) {
2021-10-13 16:48:46 +02:00
String camelCaseGetter = jsonDataPath . getParseField ( ) ! = null & & ! jsonDataPath . getParseField ( ) . isEmpty ( ) ? " get " + jsonDataPath . getParseField ( ) . substring ( 0 , 1 ) . toUpperCase ( ) + jsonDataPath . getParseField ( ) . substring ( 1 ) : " " ;
reader = aClass . getMethod ( camelCaseGetter ) ;
2020-02-10 17:24:15 +01:00
}
ObjectMapper objectMapper = new ObjectMapper ( ) ;
List < Map < String , String > > values = new ArrayList < > ( ) ;
2020-03-27 13:34:11 +01:00
int max = 1 ;
if ( reader ! = null ) {
Object invokedField = reader . invoke ( data ) ;
if ( invokedField instanceof Collection ) {
max = ( ( Collection ) invokedField ) . size ( ) ;
}
}
for ( int i = 0 ; i < max ; i + + ) {
Object value ;
if ( reader ! = null ) {
Object invokedField = reader . invoke ( data ) ;
if ( invokedField instanceof Collection ) {
value = ( ( Collection ) invokedField ) . toArray ( ) [ i ] ;
} else {
value = invokedField ;
}
} else {
value = data ;
}
Map < String , String > map = objectMapper . convertValue ( value , Map . class ) ;
if ( jsonDataPath . getMergedFields ( ) ! = null & & ! jsonDataPath . getMergedFields ( ) . isEmpty ( ) & & jsonDataPath . getMergedFieldName ( ) ! = null & & ! jsonDataPath . getMergedFieldName ( ) . isEmpty ( ) ) {
Map < String , String > finalMap = new HashMap < > ( ) ;
for ( Map . Entry < String , String > entry : map . entrySet ( ) ) {
if ( jsonDataPath . getMergedFields ( ) . contains ( entry . getKey ( ) ) ) {
if ( ! finalMap . containsKey ( jsonDataPath . getMergedFieldName ( ) ) ) {
finalMap . put ( jsonDataPath . getMergedFieldName ( ) , entry . getValue ( ) ) ;
} else {
finalMap . put ( jsonDataPath . getMergedFieldName ( ) , finalMap . get ( jsonDataPath . getMergedFieldName ( ) ) + " " + entry . getValue ( ) ) ;
}
2020-02-10 17:24:15 +01:00
} else {
2020-03-27 13:34:11 +01:00
finalMap . put ( entry . getKey ( ) , entry . getValue ( ) ) ;
2020-02-10 17:24:15 +01:00
}
}
2020-03-27 13:34:11 +01:00
values . add ( finalMap ) ;
} else {
values . add ( map ) ;
2020-02-10 17:24:15 +01:00
}
}
2024-01-03 17:22:24 +01:00
externalRefernceResult = new ExternalRefernceResult ( values , new HashMap < > ( 1 , 1 ) ) ;
2019-08-27 13:24:55 +02:00
}
2020-02-10 17:24:15 +01:00
2024-01-03 17:22:24 +01:00
if ( externalRefernceResult . getPagination ( ) . isEmpty ( ) ) {
externalRefernceResult . getPagination ( ) . put ( " pages " , 1 ) ;
externalRefernceResult . getPagination ( ) . put ( " count " , externalRefernceResult . getResults ( ) . size ( ) ) ;
2021-12-10 14:37:39 +01:00
}
2024-01-03 17:22:24 +01:00
return externalRefernceResult ;
2018-01-17 16:06:35 +01:00
}
2020-10-05 10:26:35 +02:00
} catch ( Exception exception ) {
2020-01-16 16:46:24 +01:00
logger . error ( exception . getMessage ( ) , exception ) ;
2018-01-17 16:06:35 +01:00
} //maybe print smth...
return null ;
2019-09-05 16:44:25 +02:00
}
2018-01-17 16:06:35 +01:00
2019-09-05 16:44:25 +02:00
private List < Map < String , String > > getAllResultsFromMockUpJson ( String path , String query ) {
List < Map < String , String > > internalResults ;
try {
2019-11-08 14:53:46 +01:00
String filePath = Paths . get ( path ) . toUri ( ) . toURL ( ) . toString ( ) ;
2019-09-05 16:44:25 +02:00
ObjectMapper mapper = new ObjectMapper ( ) ;
2021-06-29 13:33:57 +02:00
internalResults = mapper . readValue ( new File ( filePath ) , new TypeReference < List < Map < String , String > > > ( ) { } ) ;
2020-01-14 12:00:02 +01:00
return searchListMap ( internalResults , query ) ;
2019-09-05 16:44:25 +02:00
} catch ( Exception e ) {
2020-01-16 16:46:24 +01:00
logger . error ( e . getMessage ( ) , e ) ;
2019-09-05 16:44:25 +02:00
return new LinkedList < > ( ) ;
}
}
private List < Map < String , String > > searchListMap ( List < Map < String , String > > internalResults , String query ) {
List < Map < String , String > > list = new LinkedList < > ( ) ;
for ( Map < String , String > map : internalResults )
{
2019-09-16 17:26:18 +02:00
if ( map . get ( " name " ) ! = null & & map . get ( " name " ) . toUpperCase ( ) . contains ( query . toUpperCase ( ) ) ) {
list . add ( map ) ;
}
if ( map . get ( " label " ) ! = null & & map . get ( " label " ) . toUpperCase ( ) . contains ( query . toUpperCase ( ) ) ) {
2019-09-05 16:44:25 +02:00
list . add ( map ) ;
}
}
return list ;
2018-01-17 16:06:35 +01:00
}
2018-06-29 10:29:43 +02:00
private String transformKey ( DataUrlConfiguration dataUrlConfiguration , String key ) {
2023-05-17 12:45:12 +02:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getId ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getId ( ) . replace ( " ' " , " " ) ) ) {
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getPid ( ) = = null )
return " pid " ;
else
return " originalId " ;
}
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getPid ( ) ! = null & & key . equals ( " pid " ) ) return " pid " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getPidTypeField ( ) ! = null & & key . equals ( " pidTypeField " ) ) return " pidTypeField " ;
2019-10-01 12:19:39 +02:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getDescription ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getDescription ( ) . replace ( " ' " , " " ) ) ) return " description " ;
2019-10-03 13:06:44 +02:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getUri ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getUri ( ) . replace ( " ' " , " " ) ) ) return " uri " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getName ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getName ( ) . replace ( " ' " , " " ) ) ) return " name " ;
2019-10-01 12:19:39 +02:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getSource ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getSource ( ) . replace ( " ' " , " " ) ) ) return " source " ;
2019-10-03 13:06:44 +02:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getCount ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getCount ( ) . replace ( " ' " , " " ) ) ) return " count " ;
2020-02-10 17:24:15 +01:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getPath ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getPath ( ) . replace ( " ' " , " " ) ) ) return " path " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getHost ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getHost ( ) . replace ( " ' " , " " ) ) ) return " host " ;
2018-06-29 10:29:43 +02:00
return null ;
}
2023-10-06 17:07:58 +02:00
private String parseBodyString ( String bodyString ) {
String finalBodyString = bodyString ;
if ( bodyString . contains ( " {env: " ) ) {
int index = bodyString . indexOf ( " {env: " ) ;
while ( index > = 0 ) {
int endIndex = bodyString . indexOf ( " } " , index + 6 ) ;
String envName = bodyString . substring ( index + 6 , endIndex ) ;
finalBodyString = finalBodyString . replace ( " {env: " + envName + " } " , System . getenv ( envName ) ) ;
index = bodyString . indexOf ( " {env: " , index + 6 ) ;
}
}
return finalBodyString ;
}
2018-01-17 16:06:35 +01:00
2017-11-21 17:29:16 +01:00
}