2018-06-27 12:29:21 +02:00
package eu.eudat.logic.proxy.fetching ;
2017-11-21 17:29:16 +01:00
2019-09-05 16:44:25 +02:00
import com.fasterxml.jackson.core.type.TypeReference ;
import com.fasterxml.jackson.databind.ObjectMapper ;
2017-11-21 17:29:16 +01:00
import com.jayway.jsonpath.DocumentContext ;
import com.jayway.jsonpath.JsonPath ;
2019-10-01 12:19:39 +02:00
import eu.eudat.logic.proxy.config.* ;
2018-06-27 12:29:21 +02:00
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader ;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet ;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound ;
2018-02-16 11:34:02 +01:00
import org.springframework.beans.factory.annotation.Autowired ;
2019-09-05 16:44:25 +02:00
import org.springframework.beans.factory.annotation.Value ;
2018-02-16 11:34:02 +01:00
import org.springframework.cache.annotation.Cacheable ;
import org.springframework.stereotype.Service ;
2019-09-05 16:44:25 +02:00
import java.io.File ;
2018-02-16 11:34:02 +01:00
import java.io.IOException ;
import java.net.HttpURLConnection ;
import java.net.MalformedURLException ;
import java.net.URL ;
import java.util.* ;
2018-05-28 11:50:42 +02:00
import java.util.stream.Collectors ;
2017-11-21 17:29:16 +01:00
2017-11-22 09:57:51 +01:00
@Service
2017-11-21 17:29:16 +01:00
public class RemoteFetcher {
2018-01-17 16:06:35 +01:00
private ConfigLoader configLoader ;
2019-09-05 16:44:25 +02:00
@Value ( " ${configuration.resources.path} " )
private String resourcesPath ;
2018-01-17 16:06:35 +01:00
2018-03-28 15:24:47 +02:00
@Autowired
2019-09-05 16:44:25 +02:00
public RemoteFetcher ( ConfigLoader configLoader ) {
2018-03-28 15:24:47 +02:00
this . configLoader = configLoader ;
}
2018-01-17 16:06:35 +01:00
@Cacheable ( " repositories " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getRepositories ( String query , String key ) throws NoURLFound , HugeResultSet {
2018-06-29 10:29:43 +02:00
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getRepositories ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
2018-05-28 11:50:42 +02:00
: configLoader . getExternalUrls ( ) . getRepositories ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getRepositories ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2019-07-31 16:57:34 +02:00
@Cacheable ( " grants " )
public List < Map < String , String > > getGrants ( String query ) throws NoURLFound , HugeResultSet {
2019-09-05 16:44:25 +02:00
List < UrlConfiguration > urlConfigs = configLoader . getExternalUrls ( ) . getGrants ( ) . getUrls ( ) ;
2019-07-31 16:57:34 +02:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getGrants ( ) . getFetchMode ( ) ;
2018-01-17 16:06:35 +01:00
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2019-08-02 10:27:12 +02:00
@Cacheable ( " projects " )
public List < Map < String , String > > getProjects ( String query ) throws NoURLFound , HugeResultSet {
2019-09-05 16:44:25 +02:00
List < UrlConfiguration > urlConfigs = configLoader . getExternalUrls ( ) . getProjects ( ) . getUrls ( ) ;
2019-08-02 10:27:12 +02:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getProjects ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2019-08-20 17:22:53 +02:00
@Cacheable ( " funders " )
public List < Map < String , String > > getFunders ( String query ) throws NoURLFound , HugeResultSet {
2019-09-05 16:44:25 +02:00
List < UrlConfiguration > urlConfigs = configLoader . getExternalUrls ( ) . getFunders ( ) . getUrls ( ) ;
2019-10-01 12:19:39 +02:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getFunders ( ) . getFetchMode ( ) ;
2019-08-20 17:22:53 +02:00
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2018-01-17 16:06:35 +01:00
@Cacheable ( " organisations " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getOrganisations ( String query , String key ) throws NoURLFound , HugeResultSet {
2018-06-29 10:29:43 +02:00
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getOrganisations ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
2018-05-28 11:50:42 +02:00
: configLoader . getExternalUrls ( ) . getOrganisations ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getOrganisations ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " registries " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getRegistries ( String query , String key ) throws NoURLFound , HugeResultSet {
2018-06-29 10:29:43 +02:00
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getRegistries ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
2018-05-28 11:50:42 +02:00
: configLoader . getExternalUrls ( ) . getRegistries ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getRegistries ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " services " )
2018-06-29 10:29:43 +02:00
public List < Map < String , String > > getServices ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getServices ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
2018-05-28 11:50:42 +02:00
: configLoader . getExternalUrls ( ) . getServices ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getServices ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " researchers " )
2018-06-29 10:29:43 +02:00
public List < Map < String , String > > getResearchers ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getResearchers ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
2018-05-28 11:50:42 +02:00
: configLoader . getExternalUrls ( ) . getResearchers ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getResearchers ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2018-07-11 15:47:36 +02:00
@Cacheable ( " tags " )
public List < Map < String , String > > getTags ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getTags ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getTags ( ) . getUrls ( ) ;
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getTags ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2018-01-17 16:06:35 +01:00
@Cacheable ( " datasets " )
2018-06-29 10:29:43 +02:00
public List < Map < String , String > > getDatasets ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfiguration > urlConfigs =
key ! = null & & ! key . isEmpty ( ) ? configLoader . getExternalUrls ( ) . getDatasets ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
2018-05-28 11:50:42 +02:00
: configLoader . getExternalUrls ( ) . getDatasets ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getDatasets ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
2018-06-29 10:29:43 +02:00
private List < Map < String , String > > getAll ( List < UrlConfiguration > urlConfigs , FetchStrategy fetchStrategy , String query ) throws NoURLFound , HugeResultSet {
2018-01-17 16:06:35 +01:00
if ( urlConfigs = = null | | urlConfigs . isEmpty ( ) )
throw new NoURLFound ( " No Repository urls found in configuration " ) ;
2018-06-29 10:29:43 +02:00
Collections . sort ( urlConfigs , Comparator . comparing ( UrlConfiguration : : getOrdinal ) ) ;
List < Map < String , String > > results = new LinkedList < > ( ) ;
for ( UrlConfiguration urlConfig : urlConfigs ) {
2019-09-05 16:44:25 +02:00
if ( urlConfig . getType ( ) = = null | | urlConfig . getType ( ) . equals ( " External " ) ) {
2019-10-01 12:19:39 +02:00
results . addAll ( getAllResultsFromUrl ( urlConfig . getUrl ( ) , fetchStrategy , urlConfig . getData ( ) , urlConfig . getPaginationPath ( ) , query , urlConfig . getLabel ( ) , urlConfig . getSearch ( ) , urlConfig . getPage ( ) , urlConfig . getContentType ( ) ) ) ;
} else if ( urlConfig . getType ( ) ! = null & & urlConfig . getType ( ) . equals ( " Internal " ) ) {
2019-09-05 16:44:25 +02:00
results . addAll ( getAllResultsFromMockUpJson ( urlConfig . getUrl ( ) , query ) ) ;
}
2018-06-29 10:29:43 +02:00
}
return results ;
2018-01-17 16:06:35 +01:00
}
2019-10-01 12:19:39 +02:00
private List < Map < String , String > > getAllResultsFromUrl ( String path , FetchStrategy fetchStrategy , final DataUrlConfiguration jsonDataPath , final String jsonPaginationPath , String query , String key , DataSearchConfiguration search , DataPageConfiguration pageConfig , String contentType ) throws HugeResultSet {
Set < Integer > pages = new HashSet < > ( ) ;
2018-01-17 16:06:35 +01:00
2019-10-01 12:19:39 +02:00
String searchQuery = " " ;
Results results = null ;
if ( search ! = null & & search . getType ( ) ! = null & & ! search . getType ( ) . trim ( ) . isEmpty ( ) & &
search . getQueryParam ( ) ! = null & & ! search . getQueryParam ( ) . trim ( ) . isEmpty ( ) & &
query ! = null & & ! query . trim ( ) . isEmpty ( ) ) {
if ( search . getType ( ) . equals ( " queryParam " ) )
searchQuery = " & " + search . getQueryParam ( ) + " = " + query ;
}
2018-01-17 16:06:35 +01:00
2019-10-01 12:19:39 +02:00
if ( pageConfig ! = null ) {
if ( pageConfig . getType ( ) . equals ( " queryParam " ) )
results = getResultsFromUrl ( path + " ? " + pageConfig . getPageParam ( ) + " =1 " + searchQuery , jsonDataPath , jsonPaginationPath , contentType ) ;
} else {
results = getResultsFromUrl ( path , jsonDataPath , jsonPaginationPath , contentType ) ;
}
2018-01-17 16:06:35 +01:00
if ( fetchStrategy = = FetchStrategy . FIRST )
2019-10-01 12:19:39 +02:00
return results = = null ? new LinkedList < > ( ) : results . getResults ( ) . stream ( ) . peek ( x - > x . put ( " tag " , key ) ) . collect ( Collectors . toList ( ) ) ;
2018-01-17 16:06:35 +01:00
2019-10-01 12:19:39 +02:00
if ( results ! = null & & results . getPagination ( ) ! = null & & results . getPagination ( ) . get ( " pages " ) ! = null ) //if has more pages, add them to the pages set
2018-01-17 16:06:35 +01:00
for ( int i = 2 ; i < = results . getPagination ( ) . get ( " pages " ) ; i + + )
pages . add ( i ) ;
Long maxResults = configLoader . getExternalUrls ( ) . getMaxresults ( ) ;
2019-10-01 12:19:39 +02:00
if ( ( maxResults > 0 & & results ! = null ) & & ( results . getPagination ( ) . get ( " count " ) > maxResults ) )
2018-01-17 16:06:35 +01:00
throw new HugeResultSet ( " The submitted search query " + query + " is about to return " + results . getPagination ( ) . get ( " count " ) + " results... Please submit a more detailed search query " ) ;
2019-10-01 12:19:39 +02:00
String finalSearchQuery = searchQuery ;
2018-01-17 16:06:35 +01:00
Optional < Results > optionalResults = pages . parallelStream ( )
2019-10-01 12:19:39 +02:00
. map ( page - > getResultsFromUrl ( path + " ?page= " + page + finalSearchQuery , jsonDataPath , jsonPaginationPath , contentType ) )
2018-01-17 16:06:35 +01:00
. reduce ( ( result1 , result2 ) - > {
result1 . getResults ( ) . addAll ( result2 . getResults ( ) ) ;
return result1 ;
} ) ;
2019-10-01 12:19:39 +02:00
Results remainingResults = optionalResults . orElseGet ( Results : : new ) ;
2018-01-17 16:06:35 +01:00
remainingResults . getResults ( ) . addAll ( results . getResults ( ) ) ;
2019-10-01 12:19:39 +02:00
return remainingResults . getResults ( ) . stream ( ) . peek ( x - > x . put ( " tag " , key ) ) . collect ( Collectors . toList ( ) ) ;
2018-01-17 16:06:35 +01:00
}
2019-10-01 12:19:39 +02:00
private Results getResultsFromUrl ( String urlString , DataUrlConfiguration jsonDataPath , String jsonPaginationPath , String contentType ) {
2018-01-17 16:06:35 +01:00
try {
URL url = new URL ( urlString ) ;
HttpURLConnection con = ( HttpURLConnection ) url . openConnection ( ) ;
con . setRequestMethod ( " GET " ) ;
2019-10-01 12:19:39 +02:00
con . setRequestProperty ( " Accept " , contentType ) ;
2018-01-17 16:06:35 +01:00
int responseCode = con . getResponseCode ( ) ;
if ( responseCode = = HttpURLConnection . HTTP_OK ) { // success
//do here all the parsing
DocumentContext jsonContext = JsonPath . parse ( con . getInputStream ( ) ) ;
2019-08-27 13:24:55 +02:00
Results results ;
if ( jsonDataPath . getFieldsUrlConfiguration ( ) . getSource ( ) ! = null ) {
results = new Results ( jsonContext . read ( jsonDataPath . getPath ( )
+ " [ " + jsonDataPath . getFieldsUrlConfiguration ( ) . getName ( ) + " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getDescription ( )
+ " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getUri ( ) + " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getId ( )
+ " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getSource ( ) + " ] " ) ,
jsonContext . read ( jsonPaginationPath ) ) ;
2019-10-01 12:19:39 +02:00
} else if ( jsonDataPath . getFieldsUrlConfiguration ( ) . getCount ( ) ! = null ) { // parsing services.openair.eu
results = new Results ( jsonContext . read ( jsonDataPath . getPath ( )
+ " [ " + jsonDataPath . getFieldsUrlConfiguration ( ) . getName ( )
+ " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getId ( ) + " ] " ) ,
new HashMap < > ( 1 , 1 ) ) ;
} else {
2019-08-27 13:24:55 +02:00
results = new Results ( jsonContext . read ( jsonDataPath . getPath ( )
+ " [ " + jsonDataPath . getFieldsUrlConfiguration ( ) . getName ( ) + " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getDescription ( )
+ " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getUri ( ) + " , " + jsonDataPath . getFieldsUrlConfiguration ( ) . getId ( ) + " ] " ) ,
jsonContext . read ( jsonPaginationPath ) ) ;
}
2018-06-29 10:29:43 +02:00
results . results = results . results . stream ( ) . map ( e - > e . entrySet ( ) . stream ( ) . collect ( Collectors . toMap ( x - > this . transformKey ( jsonDataPath , x . getKey ( ) ) , Map . Entry : : getValue ) ) )
. collect ( Collectors . toList ( ) ) ;
2018-01-17 16:06:35 +01:00
return results ;
}
} catch ( MalformedURLException e1 ) {
2018-06-29 10:29:43 +02:00
e1 . printStackTrace ( ) ;
2018-01-17 16:06:35 +01:00
} //maybe print smth...
catch ( IOException e2 ) {
2018-06-29 10:29:43 +02:00
e2 . printStackTrace ( ) ;
} //maybe print smth...
catch ( Exception exception ) {
exception . printStackTrace ( ) ;
2018-01-17 16:06:35 +01:00
} //maybe print smth...
finally {
}
return null ;
2019-09-05 16:44:25 +02:00
}
2018-01-17 16:06:35 +01:00
2019-09-05 16:44:25 +02:00
private List < Map < String , String > > getAllResultsFromMockUpJson ( String path , String query ) {
String filePath = this . resourcesPath + path ;
List < Map < String , String > > internalResults ;
try {
ObjectMapper mapper = new ObjectMapper ( ) ;
internalResults = mapper . readValue ( new File ( filePath ) , new TypeReference < List < Map < String , Object > > > ( ) { } ) ;
searchListMap ( internalResults , query ) ;
return internalResults ;
} catch ( Exception e ) {
e . printStackTrace ( ) ;
return new LinkedList < > ( ) ;
}
}
private List < Map < String , String > > searchListMap ( List < Map < String , String > > internalResults , String query ) {
List < Map < String , String > > list = new LinkedList < > ( ) ;
for ( Map < String , String > map : internalResults )
{
2019-09-16 17:26:18 +02:00
if ( map . get ( " name " ) ! = null & & map . get ( " name " ) . toUpperCase ( ) . contains ( query . toUpperCase ( ) ) ) {
list . add ( map ) ;
}
if ( map . get ( " label " ) ! = null & & map . get ( " label " ) . toUpperCase ( ) . contains ( query . toUpperCase ( ) ) ) {
2019-09-05 16:44:25 +02:00
list . add ( map ) ;
}
}
return list ;
2018-01-17 16:06:35 +01:00
}
2018-06-29 10:29:43 +02:00
private String transformKey ( DataUrlConfiguration dataUrlConfiguration , String key ) {
2019-10-01 12:19:39 +02:00
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getId ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getId ( ) . replace ( " ' " , " " ) ) ) return " pid " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getDescription ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getDescription ( ) . replace ( " ' " , " " ) ) ) return " description " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getUri ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getUri ( ) . replace ( " ' " , " " ) ) ) return " uri " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getName ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getName ( ) . replace ( " ' " , " " ) ) ) return " name " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getSource ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getSource ( ) . replace ( " ' " , " " ) ) ) return " source " ;
if ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getCount ( ) ! = null & & key . equals ( dataUrlConfiguration . getFieldsUrlConfiguration ( ) . getCount ( ) . replace ( " ' " , " " ) ) ) return " count " ;
2018-06-29 10:29:43 +02:00
return null ;
}
2018-01-17 16:06:35 +01:00
2019-09-05 16:44:25 +02:00
static class Results {
2018-01-17 16:06:35 +01:00
List < Map < String , String > > results ;
Map < String , Integer > pagination ;
2019-09-05 16:44:25 +02:00
Results ( ) {
2018-01-17 16:06:35 +01:00
this . results = new ArrayList < Map < String , String > > ( ) ;
this . pagination = new HashMap < String , Integer > ( ) ;
}
2019-09-05 16:44:25 +02:00
Results ( List < Map < String , String > > results , Map < String , Integer > pagination ) {
2018-01-17 16:06:35 +01:00
this . results = results ;
this . pagination = pagination ;
}
2019-09-05 16:44:25 +02:00
List < Map < String , String > > getResults ( ) {
2018-01-17 16:06:35 +01:00
return results ;
}
public void setResults ( List < Map < String , String > > results ) {
this . results = results ;
}
2019-09-05 16:44:25 +02:00
Map < String , Integer > getPagination ( ) {
2018-01-17 16:06:35 +01:00
return pagination ;
}
public void setPagination ( Map < String , Integer > pagination ) {
this . pagination = pagination ;
}
}
2017-11-21 17:29:16 +01:00
}