2017-12-15 00:01:26 +01:00
package eu.eudat.proxy.fetching ;
2017-11-21 17:29:16 +01:00
import com.jayway.jsonpath.DocumentContext ;
import com.jayway.jsonpath.JsonPath ;
2018-03-28 15:24:47 +02:00
import eu.eudat.dynamicproject.DynamicProjectConfiguration ;
2017-12-15 00:01:26 +01:00
import eu.eudat.proxy.config.FetchStrategy ;
import eu.eudat.proxy.config.UrlConfig ;
2018-02-16 11:34:02 +01:00
import eu.eudat.proxy.config.configloaders.ConfigLoader ;
2017-12-15 00:01:26 +01:00
import eu.eudat.proxy.config.exceptions.HugeResultSet ;
import eu.eudat.proxy.config.exceptions.NoURLFound ;
2018-02-16 11:34:02 +01:00
import org.springframework.beans.factory.annotation.Autowired ;
import org.springframework.cache.annotation.Cacheable ;
import org.springframework.stereotype.Service ;
import java.io.IOException ;
import java.net.HttpURLConnection ;
import java.net.MalformedURLException ;
import java.net.URL ;
import java.util.* ;
2018-05-28 11:50:42 +02:00
import java.util.stream.Collectors ;
2017-11-21 17:29:16 +01:00
2017-11-22 09:57:51 +01:00
@Service
2017-11-21 17:29:16 +01:00
public class RemoteFetcher {
2018-01-17 16:06:35 +01:00
private ConfigLoader configLoader ;
2018-03-28 15:24:47 +02:00
private DynamicProjectConfiguration dynamicProjectConfiguration ;
2018-01-17 16:06:35 +01:00
2018-03-28 15:24:47 +02:00
@Autowired
public RemoteFetcher ( ConfigLoader configLoader , DynamicProjectConfiguration dynamicProjectConfiguration ) {
this . configLoader = configLoader ;
this . dynamicProjectConfiguration = dynamicProjectConfiguration ;
}
2018-01-17 16:06:35 +01:00
@Cacheable ( " repositories " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getRepositories ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfig > urlConfigs =
key ! = null ? configLoader . getExternalUrls ( ) . getRepositories ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getRepositories ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getRepositories ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " projects " )
public List < Map < String , String > > getProjects ( String query ) throws NoURLFound , HugeResultSet {
2018-03-28 15:24:47 +02:00
List < UrlConfig > urlConfigs = Arrays . asList ( this . dynamicProjectConfiguration . getConfiguration ( ) . getMainExternalField ( ) . getUrlConfig ( ) ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getProjects ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " organisations " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getOrganisations ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfig > urlConfigs =
key ! = null ? configLoader . getExternalUrls ( ) . getOrganisations ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getOrganisations ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getOrganisations ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " registries " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getRegistries ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfig > urlConfigs =
key ! = null ? configLoader . getExternalUrls ( ) . getRegistries ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getRegistries ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getRegistries ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " services " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getServices ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfig > urlConfigs =
key ! = null ? configLoader . getExternalUrls ( ) . getServices ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getServices ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getServices ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " researchers " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getResearchers ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfig > urlConfigs =
key ! = null ? configLoader . getExternalUrls ( ) . getResearchers ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getResearchers ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getResearchers ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
@Cacheable ( " datasets " )
2018-05-28 11:50:42 +02:00
public List < Map < String , String > > getDatasets ( String query , String key ) throws NoURLFound , HugeResultSet {
List < UrlConfig > urlConfigs =
key ! = null ? configLoader . getExternalUrls ( ) . getDatasets ( ) . getUrls ( ) . stream ( ) . filter ( item - > item . getKey ( ) . equals ( key ) ) . collect ( Collectors . toList ( ) )
: configLoader . getExternalUrls ( ) . getDatasets ( ) . getUrls ( ) ;
2018-01-17 16:06:35 +01:00
FetchStrategy fetchStrategy = configLoader . getExternalUrls ( ) . getDatasets ( ) . getFetchMode ( ) ;
return getAll ( urlConfigs , fetchStrategy , query ) ;
}
private List < Map < String , String > > getAll ( List < UrlConfig > urlConfigs , FetchStrategy fetchStrategy , String query ) throws NoURLFound , HugeResultSet {
if ( urlConfigs = = null | | urlConfigs . isEmpty ( ) )
throw new NoURLFound ( " No Repository urls found in configuration " ) ;
2018-03-28 15:24:47 +02:00
Collections . sort ( urlConfigs , Comparator . comparing ( UrlConfig : : getOrdinal ) ) ;
2018-01-17 16:06:35 +01:00
return getAllResultsFromUrl ( urlConfigs . get ( 0 ) . getUrl ( ) , fetchStrategy , urlConfigs . get ( 0 ) . getDataPath ( ) , urlConfigs . get ( 0 ) . getPaginationPath ( ) , query ) ;
}
private List < Map < String , String > > getAllResultsFromUrl ( String path , FetchStrategy fetchStrategy , final String jsonDataPath , final String jsonPaginationPath , String query ) throws HugeResultSet {
Set < Integer > pages = new HashSet < Integer > ( ) ;
final String searchQuery = ( query ! = null ) & & ! query . isEmpty ( ) ? " &search= " + query : " " ;
//first call
Results results = getResultsFromUrl ( path + " ?page=1 " + searchQuery , jsonDataPath , jsonPaginationPath ) ;
//if fetch strategy is to get only first page, then return that
if ( fetchStrategy = = FetchStrategy . FIRST )
return results = = null ? new LinkedList < > ( ) : results . getResults ( ) ;
if ( results . getPagination ( ) ! = null & & results . getPagination ( ) . get ( " pages " ) ! = null ) //if has more pages, add them to the pages set
for ( int i = 2 ; i < = results . getPagination ( ) . get ( " pages " ) ; i + + )
pages . add ( i ) ;
Long maxResults = configLoader . getExternalUrls ( ) . getMaxresults ( ) ;
if ( ( maxResults > 0 ) & & ( results . getPagination ( ) . get ( " count " ) > maxResults ) )
throw new HugeResultSet ( " The submitted search query " + query + " is about to return " + results . getPagination ( ) . get ( " count " ) + " results... Please submit a more detailed search query " ) ;
//remaining calls (if pages array has elements)
Optional < Results > optionalResults = pages . parallelStream ( )
. map ( page - > getResultsFromUrl ( path + " ?page= " + page + searchQuery , jsonDataPath , jsonPaginationPath ) )
. reduce ( ( result1 , result2 ) - > {
result1 . getResults ( ) . addAll ( result2 . getResults ( ) ) ;
return result1 ;
} ) ;
Results remainingResults = optionalResults . isPresent ( ) ? optionalResults . get ( ) : new Results ( ) ;
remainingResults . getResults ( ) . addAll ( results . getResults ( ) ) ;
return remainingResults . getResults ( ) ;
}
private Results getResultsFromUrl ( String urlString , String jsonDataPath , String jsonPaginationPath ) {
try {
URL url = new URL ( urlString ) ;
HttpURLConnection con = ( HttpURLConnection ) url . openConnection ( ) ;
con . setRequestMethod ( " GET " ) ;
con . setRequestProperty ( " Accept " , " application/vnd.api+json; charset=utf-8 " ) ;
int responseCode = con . getResponseCode ( ) ;
if ( responseCode = = HttpURLConnection . HTTP_OK ) { // success
//do here all the parsing
DocumentContext jsonContext = JsonPath . parse ( con . getInputStream ( ) ) ;
Results results = new Results ( jsonContext . read ( jsonDataPath ) , jsonContext . read ( jsonPaginationPath ) ) ;
return results ;
}
} catch ( MalformedURLException e1 ) {
} //maybe print smth...
catch ( IOException e2 ) {
} //maybe print smth...
finally {
}
return null ;
}
class Results {
List < Map < String , String > > results ;
Map < String , Integer > pagination ;
public Results ( ) {
this . results = new ArrayList < Map < String , String > > ( ) ;
this . pagination = new HashMap < String , Integer > ( ) ;
}
public Results ( List < Map < String , String > > results , Map < String , Integer > pagination ) {
this . results = results ;
this . pagination = pagination ;
}
public List < Map < String , String > > getResults ( ) {
return results ;
}
public void setResults ( List < Map < String , String > > results ) {
this . results = results ;
}
public Map < String , Integer > getPagination ( ) {
return pagination ;
}
public void setPagination ( Map < String , Integer > pagination ) {
this . pagination = pagination ;
}
}
2017-11-21 17:29:16 +01:00
}