From 5cc9343aebad49ddb280a796f1b007953535dbdf Mon Sep 17 00:00:00 2001 From: Francesco Mangiacrapa Date: Tue, 21 May 2019 14:27:58 +0000 Subject: [PATCH] on going on bug fixing the Incident #16671 git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/uri-resolver@179493 82a268e6-3cf1-43bd-a215-b396298e98cf --- .settings/org.eclipse.wst.common.component | 3 - .../LoadingGNPublicLayerIDsInstanceCache.java | 116 +++++++++ .../LoadingGeonetworkInstanceCache.java | 9 +- .../gis/GeonetworkAccessParameter.java | 62 ++--- .../resolver/gis/GeonetworkInstance.java | 80 +++++-- .../gis/GeonetworkServiceInterface.java | 20 +- .../gis/geonetwork/FilterGetRecords.java | 96 +++----- .../resolver/services/CatalogueResolver.java | 2 +- .../resolver/services/GeonetworkResolver.java | 226 +++++++++++------- .../util/GetResponseRecordFilter.java | 164 ++++++++++--- 10 files changed, 501 insertions(+), 277 deletions(-) create mode 100644 src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGNPublicLayerIDsInstanceCache.java diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index f50af13..cfa6374 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -4,9 +4,6 @@ - - uses - diff --git a/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGNPublicLayerIDsInstanceCache.java b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGNPublicLayerIDsInstanceCache.java new file mode 100644 index 0000000..d2e0e75 --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGNPublicLayerIDsInstanceCache.java @@ -0,0 +1,116 @@ +/** + * + */ + +package org.gcube.datatransfer.resolver.caches; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import org.gcube.spatial.data.geonetwork.GeoNetwork; +import org.gcube.spatial.data.geonetwork.GeoNetworkAdministration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; + +import it.geosolutions.geonetwork.util.GNSearchRequest; +import it.geosolutions.geonetwork.util.GNSearchResponse; +import it.geosolutions.geonetwork.util.GNSearchResponse.GNMetadata; + + + +/** + * The Class LoadingGNPublicLayerIDsInstanceCache. + * + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * + * May 21, 2019 + */ +public class LoadingGNPublicLayerIDsInstanceCache { + + private static Logger logger = LoggerFactory.getLogger(LoadingGNPublicLayerIDsInstanceCache.class); + + private static LoadingCache> gnPublicLayersCache; + + static { + + CacheLoader> loader = new CacheLoader> () { + @Override + public List load(String geonetworkEndPoint) + throws Exception { + //logger.info("Loading public layer IDS for GN endpoint: "+geonetworkEndPoint); + return loadGNPublicLayersID(geonetworkEndPoint); + } + }; + + RemovalListener> removalListener = new RemovalListener>() { + public void onRemoval(RemovalNotification> removal) { + logger.info("cache expired"); + } + }; + + gnPublicLayersCache = + CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite( + 1, TimeUnit.DAYS).removalListener(removalListener). + build(loader); + + logger.info("cache instancied"); + } + + + /** + * Gets the. + * + * @param scope the scope + * @return the geonetwork instance + * @throws ExecutionException the execution exception + */ + public static List get(String scope) throws ExecutionException{ + + return gnPublicLayersCache.get(scope); + } + + + /** + * Load GN public layers ID. + * + * @param geonetworkEndPoint the geonetwork end point + * @return the list + * @throws Exception the exception + */ + protected static List loadGNPublicLayersID(String geonetworkEndPoint) + throws Exception { + + if (geonetworkEndPoint == null || geonetworkEndPoint.isEmpty()){ + logger.warn("geonetworkEndPoint is null or empty, returning null"); + return null; + } + + List foundPublicIds = new ArrayList(); + try { + logger.info("Loading Public Layers ID for GN endpoint: {}",geonetworkEndPoint); + GeoNetworkAdministration reader = GeoNetwork.get(); + final GNSearchRequest req=new GNSearchRequest(); + req.addParam(GNSearchRequest.Param.any,""); + GNSearchResponse resp=reader.query(req); + + Iterator iterator=resp.iterator(); + while(iterator.hasNext()){ + foundPublicIds.add(iterator.next().getUUID()); + } + logger.info("Public Layers ID are: "+foundPublicIds.size()); + }catch (Exception e) { + logger.error("Error during sending GNSearchRequest: ",e); + } + + return foundPublicIds; + } +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGeonetworkInstanceCache.java b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGeonetworkInstanceCache.java index 4c587e5..a260ac5 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGeonetworkInstanceCache.java +++ b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingGeonetworkInstanceCache.java @@ -8,6 +8,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter; +import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.AccountType; import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.GeonetworkLoginLevel; import org.gcube.datatransfer.resolver.gis.GeonetworkInstance; import org.gcube.datatransfer.resolver.gis.exception.GeonetworkInstanceException; @@ -52,7 +53,7 @@ public class LoadingGeonetworkInstanceCache { }; geonetworkInstancesCache = - CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite( + CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite( 1, TimeUnit.DAYS).removalListener(removalListener). build(loader); @@ -89,8 +90,8 @@ public class LoadingGeonetworkInstanceCache { } GeonetworkAccessParameter gntwAccess = new GeonetworkAccessParameter(scope); - GeonetworkInstance gnInstance = gntwAccess.getGeonetworkInstance(true, GeonetworkLoginLevel.ADMIN); - logger.info("Loaded "+gnInstance+" for scope: " + scope); - return gnInstance; + GeonetworkInstance geoInstance = gntwAccess.getGeonetworkInstance(false, GeonetworkLoginLevel.CKAN, AccountType.CKAN); + logger.info("Loaded "+geoInstance+" for scope: " + scope); + return geoInstance; } } diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkAccessParameter.java b/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkAccessParameter.java index a364d47..627bb31 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkAccessParameter.java +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkAccessParameter.java @@ -10,8 +10,9 @@ import org.slf4j.LoggerFactory; /** * The Class GeonetworkAccessParameter. * - * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it - * Jun 9, 2016 + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * + * May 20, 2019 */ public class GeonetworkAccessParameter implements GeonetworkServiceInterface{ @@ -57,58 +58,27 @@ public class GeonetworkAccessParameter implements GeonetworkServiceInterface{ public GeonetworkAccessParameter(String scope) { this.scope = scope; } - + /* (non-Javadoc) - * @see org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface#getGeonetworkInstance(boolean, org.gcube.datatransfer.resolver.gis.GeonetowrkAccessParameter.GeonetworkLoginLevel) + * @see org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface#getGeonetworkInstance(boolean, org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.GeonetworkLoginLevel, org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.AccountType) */ - public GeonetworkInstance getGeonetworkInstance(boolean authenticate, GeonetworkLoginLevel loginLevel) throws GeonetworkInstanceException { - return instanceGeonetwork(authenticate, loginLevel, null); - } + public GeonetworkInstance getGeonetworkInstance(boolean authenticate, GeonetworkLoginLevel loginLevel, AccountType accType) throws GeonetworkInstanceException{ + + if(geonetworkInstance==null) { - /** - * Instance geonetwork. - * - * @param authenticate the authenticate - * @param loginLevel the login level - * @param accType the acc type - * @return the geonetwork instance - * @throws GeonetworkInstanceException the geonetwork instance exception - */ - private GeonetworkInstance instanceGeonetwork(boolean authenticate, GeonetworkLoginLevel loginLevel, AccountType accType) throws GeonetworkInstanceException{ - - if(scope == null || scope.isEmpty()) - throw new GeonetworkInstanceException("Scope is null"); - - LoginLevel level = toLoginLevel(loginLevel); - Type type = toType(accType); - if(geonetworkInstance==null) + if(scope == null || scope.isEmpty()) + throw new GeonetworkInstanceException("Scope is null"); + + LoginLevel level = loginLevel!=null?toLoginLevel(loginLevel):null; + Type type = accType!=null?toType(accType):null; geonetworkInstance = new GeonetworkInstance(scope, authenticate, level, type); - - return geonetworkInstance; - - } - - - /* (non-Javadoc) - * @see org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface#getGeonetworkInstance() - */ - @Override - public GeonetworkInstance getGeonetworkInstance() - throws Exception { - - if(scope == null || scope.isEmpty()) - throw new GeonetworkInstanceException("Scope is null"); - - /*if(serverParam.getUrl() == null || serverParam.getUrl().isEmpty()) - throw new GeonetworkInstanceException("Geonetwork url is null or empty");*/ - - if(geonetworkInstance==null) - geonetworkInstance = new GeonetworkInstance(scope, false, null, null); - + } + return geonetworkInstance; } + /* (non-Javadoc) * @see org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface#getScope() */ diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkInstance.java b/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkInstance.java index 3e32b91..66f5bbc 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkInstance.java +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkInstance.java @@ -6,6 +6,7 @@ import org.gcube.spatial.data.geonetwork.GeoNetwork; import org.gcube.spatial.data.geonetwork.GeoNetworkPublisher; import org.gcube.spatial.data.geonetwork.LoginLevel; import org.gcube.spatial.data.geonetwork.configuration.Configuration; +import org.gcube.spatial.data.geonetwork.extension.ServerAccess.Version; import org.gcube.spatial.data.geonetwork.model.Account; import org.gcube.spatial.data.geonetwork.model.Account.Type; import org.gcube.spatial.data.geonetwork.model.faults.AuthorizationException; @@ -14,39 +15,25 @@ import org.gcube.spatial.data.geonetwork.model.faults.MissingServiceEndpointExce import org.slf4j.Logger; import org.slf4j.LoggerFactory; - - - /** * The Class GeonetworkInstance. * - * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it - * May 16, 2017 + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * + * May 21, 2019 */ public class GeonetworkInstance { private GeoNetworkPublisher geonetworkPublisher = null; - /** - * Instantiates a new geonetwork instance. - */ - public GeonetworkInstance(){} //FOR SERIALIZATION - private Logger logger = LoggerFactory.getLogger(GeonetworkInstance.class); private String scope; private Account account; private LoginLevel level; private Type type; - - /** - * Instantiates a new geonetwork instance. - * - * @param scope the scope - * @throws GeonetworkInstanceException the geonetwork instance exception - */ - public GeonetworkInstance(String scope) throws GeonetworkInstanceException{ - this(scope, false, null, null); - } + private Configuration config; + private Version version; + private String endPoint; /** * Instantiates a new geonetwork instance. @@ -67,9 +54,11 @@ public class GeonetworkInstance { ScopeProvider.instance.set(scope); logger.info("setting scope "+scope); createInstanceGeonetworkPublisher(authenticate, level); + this.config = geonetworkPublisher.getConfiguration(); + this.version = this.config.getGeoNetworkVersion(); + this.endPoint = this.config.getGeoNetworkEndpoint(); if(this.type!=null){ - Configuration config = geonetworkPublisher.getConfiguration(); - this.account=config.getScopeConfiguration().getAccounts().get(type); + this.account=config.getScopeConfiguration().getAccounts().get(this.type); } //logger.info("Admin: "+config.getAdminAccount().getUser()+", Pwd: "+config.getAdminAccount().getPassword()); } catch (Exception e) { @@ -140,7 +129,26 @@ public class GeonetworkInstance { public GeoNetworkPublisher getGeonetworkPublisher() { return geonetworkPublisher; } - + + + /** + * Gets the version. + * + * @return the version + */ + public Version getVersion() { + return version; + } + + + /** + * Gets the level. + * + * @return the level + */ + public LoginLevel getLevel() { + return level; + } /** * Gets the account. * @@ -150,6 +158,25 @@ public class GeonetworkInstance { return account; } + + + /** + * Gets the end point. + * + * @return the end point + */ + public String getEndPoint() { + return endPoint; + } + + /** + * Gets the config. + * + * @return the config + */ + public Configuration getConfig() { + return config; + } /** * Gets the scope. @@ -166,12 +193,9 @@ public class GeonetworkInstance { */ @Override public String toString() { - StringBuilder builder = new StringBuilder(); builder.append("GeonetworkInstance [geonetworkPublisher="); builder.append(geonetworkPublisher); - builder.append(", logger="); - builder.append(logger); builder.append(", scope="); builder.append(scope); builder.append(", account="); @@ -180,8 +204,12 @@ public class GeonetworkInstance { builder.append(level); builder.append(", type="); builder.append(type); + builder.append(", config="); + builder.append(config); builder.append("]"); return builder.toString(); } + + } diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkServiceInterface.java b/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkServiceInterface.java index d401b7e..7c8b8b2 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkServiceInterface.java +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/GeonetworkServiceInterface.java @@ -3,6 +3,7 @@ */ package org.gcube.datatransfer.resolver.gis; +import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.AccountType; import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.GeonetworkLoginLevel; /** @@ -17,20 +18,21 @@ public interface GeonetworkServiceInterface { * * @param authenticate the authenticate * @param loginLevel the login level + * @param accType the acc type * @return the geonetwork instance * @throws Exception the exception */ - public GeonetworkInstance getGeonetworkInstance(boolean authenticate, GeonetworkLoginLevel loginLevel) throws Exception; + public GeonetworkInstance getGeonetworkInstance(boolean authenticate, GeonetworkLoginLevel loginLevel, AccountType accType) throws Exception; - - /** - * Gets the geonetwork instance with authenticate = false and LoginLevel = null - * - * @return the geonetwork instance - * @throws Exception the exception - */ - public GeonetworkInstance getGeonetworkInstance() throws Exception; +// +// /** +// * Gets the geonetwork instance with authenticate = false and LoginLevel = null. +// * +// * @return the geonetwork instance +// * @throws Exception the exception +// */ +// public GeonetworkInstance getGeonetworkInstance() throws Exception; /** * Gets the scope. diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java b/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java index ad2a494..33f8531 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java @@ -3,19 +3,12 @@ */ package org.gcube.datatransfer.resolver.gis.geonetwork; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import org.gcube.spatial.data.geonetwork.GeoNetwork; -import org.gcube.spatial.data.geonetwork.GeoNetworkReader; +import org.gcube.datatransfer.resolver.caches.LoadingGNPublicLayerIDsInstanceCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import it.geosolutions.geonetwork.util.GNSearchRequest; -import it.geosolutions.geonetwork.util.GNSearchResponse; -import it.geosolutions.geonetwork.util.GNSearchResponse.GNMetadata; - /** * The Class FilterGetRecords. @@ -32,78 +25,53 @@ public class FilterGetRecords { private static final Logger logger = LoggerFactory.getLogger(FilterGetRecords.class); - private List foundPublicIds = null; + private List foundPublicLayerIds = null; + + private String geonetworkEndPoint; /** * Instantiates a new filter get records. * * @param readBody the read body + * @param geonetworkEndPoint the geonetwork end point */ - public FilterGetRecords(String readBody) { + public FilterGetRecords(String readBody, String geonetworkEndPoint) { + this.geonetworkEndPoint = geonetworkEndPoint; if(readBody!=null && !readBody.isEmpty() && readBody.contains(CSW_GET_RECORDS)){ - logger.info("Is "+CSW_GET_RECORDS+" request, getting public ids"); - GeoNetworkReader reader; - try { - reader = GeoNetwork.get(); - final GNSearchRequest req=new GNSearchRequest(); - req.addParam(GNSearchRequest.Param.any,""); - GNSearchResponse resp=reader.query(req); - - foundPublicIds = new ArrayList(); - Iterator iterator=resp.iterator(); - while(iterator.hasNext()){ - foundPublicIds.add(iterator.next().getUUID()); - } - logger.info("Public Metadata ids are: "+foundPublicIds.size()); - }catch (Exception e) { - logger.error("Error during sending GNSearchRequest: ",e); - } + logger.info("The request is "+CSW_GET_RECORDS+" so getting GN public layer IDs"); + loadGNPublicLayers(); }else logger.trace("Is not a"+CSW_GET_RECORDS+" request, skipping"); } - - - - /** - * Gets the public file identifiers. - * - * @return the public file identifiers - */ - public List getPublicFileIdentifiers(){ - - logger.info("Performing query to retrieve the public file identifiers"); - GeoNetworkReader reader; - try { - reader = GeoNetwork.get(); - final GNSearchRequest req=new GNSearchRequest(); - req.addParam(GNSearchRequest.Param.any,""); - GNSearchResponse resp=reader.query(req); - - foundPublicIds = new ArrayList(); - Iterator iterator=resp.iterator(); - while(iterator.hasNext()){ - foundPublicIds.add(iterator.next().getUUID()); - } - logger.info("Public Metadata ids are: "+foundPublicIds.size()); - - }catch (Exception e) { - logger.error("Error during performing "+CSW_GET_RECORDS+": ",e); - } - - return foundPublicIds; - } - - + + /** * Gets the found public ids. * - * @return the foundPublicIds + * @return the found public ids */ public List getFoundPublicIds() { - - return foundPublicIds==null || foundPublicIds.isEmpty()? null: foundPublicIds; + + if(foundPublicLayerIds==null) { + loadGNPublicLayers(); + } + + if(foundPublicLayerIds.isEmpty()) { + return null; + } + + return foundPublicLayerIds; + } + + private void loadGNPublicLayers(){ + try { + foundPublicLayerIds = LoadingGNPublicLayerIDsInstanceCache.get(geonetworkEndPoint); + logger.info("For the GN {}, I found {} public ID layer/s",geonetworkEndPoint,foundPublicLayerIds.size()); + }catch (Exception e) { + logger.error("Error occurred on loading cache of GN public IDs: ",e); + } } @@ -115,7 +83,7 @@ public class FilterGetRecords { StringBuilder builder = new StringBuilder(); builder.append("FilterGetRecords [foundPublicIds="); - builder.append(foundPublicIds); + builder.append(foundPublicLayerIds); builder.append("]"); return builder.toString(); } diff --git a/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java b/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java index 839aa62..4a06ffe 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java +++ b/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java @@ -18,8 +18,8 @@ import javax.ws.rs.core.Response; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; -import org.gcube.datatransfer.resolver.caches.LoadingMapOfScopeCache; import org.gcube.datatransfer.resolver.ConstantsResolver; +import org.gcube.datatransfer.resolver.caches.LoadingMapOfScopeCache; import org.gcube.datatransfer.resolver.catalogue.CatalogueRequest; import org.gcube.datatransfer.resolver.catalogue.ItemCatalogueURLs; import org.gcube.datatransfer.resolver.catalogue.ResourceCatalogueCodes; diff --git a/src/main/java/org/gcube/datatransfer/resolver/services/GeonetworkResolver.java b/src/main/java/org/gcube/datatransfer/resolver/services/GeonetworkResolver.java index d99589f..3ad6657 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/services/GeonetworkResolver.java +++ b/src/main/java/org/gcube/datatransfer/resolver/services/GeonetworkResolver.java @@ -26,10 +26,10 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.datatransfer.resolver.ConstantsResolver; -import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter; +import org.gcube.datatransfer.resolver.caches.LoadingGeonetworkInstanceCache; +import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.AccountType; +import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter.GeonetworkLoginLevel; import org.gcube.datatransfer.resolver.gis.GeonetworkInstance; -import org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface; -import org.gcube.datatransfer.resolver.gis.exception.GeonetworkInstanceException; import org.gcube.datatransfer.resolver.gis.geonetwork.FilterGetRecords; import org.gcube.datatransfer.resolver.gis.geonetwork.GNAuthentication; import org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkRequestFilterParameters.MODE; @@ -42,6 +42,7 @@ import org.gcube.datatransfer.resolver.util.HTTPCallsUtils.HttpResponse; import org.gcube.datatransfer.resolver.util.ScopeUtil; import org.gcube.datatransfer.resolver.util.SingleFileStreamingOutput; import org.gcube.spatial.data.geonetwork.configuration.Configuration; +import org.gcube.spatial.data.geonetwork.extension.ServerAccess.Version; import org.gcube.spatial.data.geonetwork.model.Account; import org.gcube.spatial.data.geonetwork.model.Account.Type; import org.slf4j.Logger; @@ -85,7 +86,7 @@ public class GeonetworkResolver { public static final String SCOPE_SEPARATOR = "|"; - protected Map cacheGNInstances; + //protected Map cacheGNInstances; private String helpURI = "https://wiki.gcube-system.org/gcube/GCube_Resource_Catalogue#Geonetwork_Resolver"; @@ -100,14 +101,16 @@ public class GeonetworkResolver { * OWNER (is optional): filter by owner * * @param req the req - * @param mode the mode * @param scope the scope + * @param mode the mode * @param visibility the visibility - * @param owner the owner - pass 'null' as string if no filter applied - * @param requestDelimiter the request delimiter + * @param filterKey the filter key + * @param filterValue the filter value + * @param remainPath the remain path * @param resetCache the reset cache * @param resetScope the reset scope * @return the geonetwork request criteria + * @throws WebApplicationException the web application exception */ @GET @@ -162,28 +165,62 @@ public class GeonetworkResolver { throw ExceptionManager.wrongParameterException(req, "The 'visibility' parameter must be value of "+toPrint, this.getClass(), helpURI); } - if(resetCache!=null && Boolean.parseBoolean(resetCache)){ - purgeCacheGeonetworkInstances(); - } - - if(resetScope!=null && Boolean.parseBoolean(resetScope)){ - resetGeonetoworkInstanceCacheForScope(scope); - } - logger.info("Remaining path is: "+remainPath); try { - - GeonetworkInstance gnInstance = getGeonetworkInstanceForScope(scope); - + + //I'M LOADING THE GN CONFIGURATIONS (ENDPOINT, USER, PWD AND SO ON..) FROM GN LIBRARY BY GENERAL CONSTRUCTOR THAT PERFORMS AUTHENTICATION ON GN, + //THEN THE CONGIGURATIONS LAODED, USER TO PERFORM AUTHENTICATION AND SO ON ARE MANAGE VIA HTTP_CLIENTS. + //I'M LOADING THE GN CONFIGURATIONS (ENDPOINT, USER, PWD AND SO ON..) FROM GN LIBRARY BY GENERAL CONSTRUCTOR THAT PERFORMS AUTHENTICATION ON GN, + //THEN THE CONGIGURATIONS LAODED, USER TO PERFORM AUTHENTICATION AND SO ON ARE MANAGE VIA HTTP_CLIENTS. + logger.info("set scope provider "+scope); ScopeProvider.instance.set(scope); - + GeonetworkInstance gnInstance = getGeonetworkInstanceForScope(scope, GeonetworkLoginLevel.CKAN, AccountType.CKAN); + Account account = gnInstance.getAccount(); + Version version = gnInstance.getVersion(); + String geonetworkUrl = gnInstance.getEndPoint(); + Configuration config = gnInstance.getConfig()!=null?gnInstance.getConfig():null; + + if(account==null || account.getUser()==null || account.getPassword()==null || config==null) { + logger.info("Loading GN istance and configurations via Geonetwork Library..."); + logger.info("set scope provider "+scope); + ScopeProvider.instance.set(scope); + config = gnInstance.getGeonetworkPublisher().getConfiguration(); + account = config.getScopeConfiguration().getAccounts().get(Type.CKAN); + version = config.getGeoNetworkVersion(); + geonetworkUrl = config.getGeoNetworkEndpoint(); + } + //logger.info("set scope provider "+scope); + //ScopeProvider.instance.set(scope); + logger.info("SCOPE: {}, CKAN user owner is: {}",scope, account.getUser()); + logger.info("SCOPE: {}, GN EndPoint: {}",scope, geonetworkUrl); + +// ScopeProvider.instance.set(scope); HTTPCallsUtils httpUtils = new HTTPCallsUtils(); - Configuration config = gnInstance.getGeonetworkPublisher().getConfiguration(); - String geonetworkUrl = config.getGeoNetworkEndpoint(); +// Configuration config = gnInstance.getGeonetworkPublisher().getConfiguration(); +// String geonetworkUrl = config.getGeoNetworkEndpoint(); String baseURL = remainPath==null ||remainPath.isEmpty()?geonetworkUrl+"/"+CSW_SERVER:geonetworkUrl+"/"+CSW_SERVER+remainPath; logger.info("The base URL is: "+baseURL); String queryString = req.getQueryString()==null || req.getQueryString().isEmpty()?"":"?"+req.getQueryString(); gnGetlURL = baseURL+queryString; + +// Account account = config.getScopeConfiguration().getAccounts().get(Type.CKAN); + + if(account.getUser()!=null){ + switch (version) { + case DUE: + boolean authorized = GNAuthentication.login(httpUtils, geonetworkUrl, account.getUser(), account.getPassword()); + logger.info("Authorized on GN2 "+geonetworkUrl +" ? "+authorized); + break; + + default: + httpUtils = new HTTPCallsUtils(account.getUser(), account.getPassword()); + logger.info("Authorized on GN3 via HTTCallsUtils..."); + break; + } + }else { + logger.warn("I'm not able to perform authentication, the user read from config with "+Type.CKAN+" is null"); + } + logger.info("Sending get request to URL: "+gnGetlURL); HttpResponse proxedGNResponse = httpUtils.get(gnGetlURL); @@ -228,6 +265,21 @@ public class GeonetworkResolver { } + /** + * Submit post. + * + * @param req the req + * @param scope the scope + * @param mode the mode + * @param visibility the visibility + * @param filterKey the filter key + * @param filterValue the filter value + * @param remainPath the remain path + * @param resetCache the reset cache + * @param resetScope the reset scope + * @return the response + * @throws WebApplicationException the web application exception + */ @POST @Path("/{"+PATH_PARAM_SCOPE+"}/{"+PATH_PARAM_MODE+"}/{"+PATH_PARAM_VISIBILITY+"}/{filterKey}/{filterValue}/$${"+PATH_PARAM_REMAINPATH+":(/[^?$]+)?}") public Response submitPost(@Context HttpServletRequest req, @@ -294,14 +346,34 @@ public class GeonetworkResolver { try { - GeonetworkServiceInterface gntwAccess = new GeonetworkAccessParameter(scope); - GeonetworkInstance gnInstance = gntwAccess.getGeonetworkInstance(); - - ScopeProvider.instance.set(scope); + //MANUALLY INSTANCING THE GEONETWORK TO MANGE LOGIN IF NEEDED AND SO ON.. + //DO NOT USE THE LOADING_CACHE OF GEONETWORK +// GeonetworkServiceInterface gntwAccess = new GeonetworkAccessParameter(scope); +// GeonetworkInstance gnInstance = gntwAccess.getGeonetworkInstance(false, null, null); + + //I'M LOADING THE GN CONFIGURATIONS (ENDPOINT, USER, PWD AND SO ON..) FROM GN LIBRARY BY GENERAL CONSTRUCTOR THAT PERFORMS AUTHENTICATION ON GN, + //THEN THE CONGIGURATIONS LAODED, USER TO PERFORM AUTHENTICATION AND SO ON ARE MANAGE VIA HTTP_CLIENTS. logger.info("set scope provider "+scope); - Configuration config = gnInstance.getGeonetworkPublisher().getConfiguration(); - Account account = config.getScopeConfiguration().getAccounts().get(Type.CKAN); - logger.info("CKAN user owner is: "+account.getUser()); + ScopeProvider.instance.set(scope); + GeonetworkInstance gnInstance = getGeonetworkInstanceForScope(scope, GeonetworkLoginLevel.CKAN, AccountType.CKAN); + Account account = gnInstance.getAccount(); + Version version = gnInstance.getVersion(); + String geonetworkUrl = gnInstance.getEndPoint(); + Configuration config = gnInstance.getConfig()!=null?gnInstance.getConfig():null; + + if(account==null || account.getUser()==null || account.getPassword()==null || config==null) { + logger.info("Loading GN istance and configurations via Geonetwork Library..."); + logger.info("set scope provider "+scope); + ScopeProvider.instance.set(scope); + config = gnInstance.getGeonetworkPublisher().getConfiguration(); + account = config.getScopeConfiguration().getAccounts().get(Type.CKAN); + version = config.getGeoNetworkVersion(); + geonetworkUrl = config.getGeoNetworkEndpoint(); + } + logger.info("set scope provider "+scope); + ScopeProvider.instance.set(scope); + logger.info("SCOPE: {}, CKAN user used is: {}",scope, account.getUser(), account.getPassword()); + logger.info("SCOPE: {}, GN EndPoint: {}",scope, geonetworkUrl); // logger.info("Parameters.."); // for (Enumeration e = req.getParameterNames(); e.hasMoreElements();){ @@ -314,7 +386,6 @@ public class GeonetworkResolver { // logger.debug("doPost read body request: "+readBody); ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); - String geonetworkUrl = config.getGeoNetworkEndpoint(); // SPECIFIC HANDLER FOR GEONETWORK REQUEST: /srv/en/mef.export if(remainPath!=null && remainPath.compareTo(SRV_EN_MEF_EXPORT)==0){ @@ -345,7 +416,7 @@ public class GeonetworkResolver { if(visibility.equals(VISIBILITY.PRV.name())){ logger.info("Visibility: "+VISIBILITY.PRV+" getting private layers.."); //VRE LAYERS - if(mode.equals(MODE.VRE)){ + if(mode.equals(MODE.VRE.name())){ logger.info("Getting "+MODE.VRE+" layers.."); //HARVESTED LAYERS }else{ @@ -354,10 +425,21 @@ public class GeonetworkResolver { } if(account.getUser()!=null){ - boolean authorized = GNAuthentication.login(httpUtils, geonetworkUrl, account.getUser(), account. getPassword()); - logger.info("Authorized on "+geonetworkUrl +" ? "+authorized); - }else - logger.info("Skipping authentication, ckan user (the owner) is null"); + switch (version) { + case DUE: + boolean authorized = GNAuthentication.login(httpUtils, geonetworkUrl, account.getUser(), account.getPassword()); + logger.info("Authorized on GN2 "+geonetworkUrl +" ? "+authorized); + break; + + default: + httpUtils = new HTTPCallsUtils(account.getUser(), account.getPassword()); + logger.info("Authorized on GN3 via HTTCallsUtils..."); + break; + } + + }else { + logger.warn("I'm not able to perform authentication, the user read from config with "+Type.CKAN+" is null"); + } //PUBLIC LAYERS }else{ @@ -391,7 +473,7 @@ public class GeonetworkResolver { if(visibility.equals(VISIBILITY.PRV.name())){ logger.info("Private VISIBILITY requested, retrieving public file identifiers to apply filtering.."); - FilterGetRecords filterGetRecords = new FilterGetRecords(byteArray.toString()); + FilterGetRecords filterGetRecords = new FilterGetRecords(byteArray.toString(), geonetworkUrl); if(filterGetRecords.getFoundPublicIds()!=null && filterGetRecords.getFoundPublicIds().size()>0){ logger.info("I'm removing list of public IDs with "+filterGetRecords.getFoundPublicIds().size() +" item/s. Is it right?"); in = GetResponseRecordFilter.overrideResponseIdsByListIds(reus, filterGetRecords.getFoundPublicIds(), REPLACED_A_PUBLIC_UUID_PLEASE_IGNORE); @@ -475,73 +557,31 @@ public class GeonetworkResolver { } } - - /** - * Discovery geonetwork instance. - * - * @param scope the scope - * @return the geonetwork instance - * @throws GeonetworkInstanceException the geonetwork instance exception - */ - private GeonetworkInstance discoveryGeonetworkInstance(String scope) throws GeonetworkInstanceException{ - - GeonetworkAccessParameter gntwAccess = new GeonetworkAccessParameter(scope); - return gntwAccess.getGeonetworkInstance(true, null); - } - - /** - * Reset geonetowork instance cache for scope. - * - * @param scope the scope - */ - private void resetGeonetoworkInstanceCacheForScope(String scope){ - if(cacheGNInstances!=null && cacheGNInstances.get(scope)!=null){ - cacheGNInstances.remove(scope); - logger.info("Reset of "+scope+" in Cache Geonetwork server params perfomed!"); - }else - logger.info("Reset of "+scope+" in Cache Geonetwork skipped, scope not exists!"); - } - - - - /** - * Purge cache geonetwork instances. - */ - private void purgeCacheGeonetworkInstances(){ - cacheGNInstances = new HashMap(); - logger.info("Reset of GeonetworkInstance cache perfomed!"); - } - + /** + * + * !!!!!USE ONLY IN GET METHOD + * * Gets the geonetwork instance for scope. * * @param scope the scope + * @param loginLevel the login level + * @param accountType the account type * @return the geonetwork instance for scope * @throws Exception the exception */ - protected GeonetworkInstance getGeonetworkInstanceForScope(String scope) throws Exception{ - - if(cacheGNInstances==null) - purgeCacheGeonetworkInstances(); - - scope = ScopeUtil.normalizeScope(scope, "|"); - logger.info("Attempt to get geonetwork instance from GeonetworkInstance cache for scope: "+scope); - GeonetworkInstance geoInstance = cacheGNInstances.get(scope); - - if(geoInstance==null){ - logger.info("Cache having null GeonetworkInstance for scope "+scope+", reading by Geonetwork library..."); - try { - geoInstance = discoveryGeonetworkInstance(scope); - cacheGNInstances.put(scope, geoInstance); - logger.info("Updated GeonetworkInstance Cache adding couple: Scope "+scope+" - GeonetworkInstance "+geoInstance); - } catch (Exception e) { - logger.error("An error occurred on reading GeonetworkInstance for scope "+scope, e); - throw new Exception("Sorry, An error occurred on reading GeonetworkInstance for scope "+scope); - } - }else - logger.info("GeonetworkInstance cache for scope: "+scope+" is not null using it: "+geoInstance); + protected GeonetworkInstance getGeonetworkInstanceForScope(String scope, GeonetworkLoginLevel loginLevel, AccountType accountType) throws Exception{ + logger.info("Trying to read the {} from cache for scope: {}",GeonetworkInstance.class.getSimpleName(), scope); + GeonetworkInstance geoInstance; + try { + geoInstance = LoadingGeonetworkInstanceCache.get(scope); + }catch (Exception e) { + logger.error("An error occurred on reading GeonetworkInstance for scope "+scope, e); + throw new Exception("Sorry, An error occurred on reading GeonetworkInstance for scope "+scope); + } return geoInstance; + } } diff --git a/src/main/java/org/gcube/datatransfer/resolver/util/GetResponseRecordFilter.java b/src/main/java/org/gcube/datatransfer/resolver/util/GetResponseRecordFilter.java index e93ddba..56294ed 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/util/GetResponseRecordFilter.java +++ b/src/main/java/org/gcube/datatransfer/resolver/util/GetResponseRecordFilter.java @@ -13,8 +13,10 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -125,13 +127,14 @@ public class GetResponseRecordFilter { * @param doc the doc * @param identifier the identifier * @param messageToReplace the message to replace - * @return true, if successful + * @return the map */ - private static boolean overrideSummaryRecord(Document doc, String identifier, String messageToReplace) { + private static boolean overrideSummaryRecord(Document doc, String identifier, String messageToReplace, Map overridingRecordsMap) { // list NodeList nodes = doc.getElementsByTagName("gmd:MD_Metadata"); logger.trace("gmd:MD_Metadata are: " + nodes.getLength()); + for (int i = 0; i < nodes.getLength(); i++) { Element mdMetadata = (Element) nodes.item(i); @@ -152,14 +155,22 @@ public class GetResponseRecordFilter { Element gco = (Element) gcoLst.item(0); String idValue = gco.getTextContent(); + + if(overridingRecordsMap.get(idValue)==null) { + overridingRecordsMap.put(idValue, false); + } + logger.trace("Summary gmd:fileIdentifier is: " + idValue); - if (idValue!=null && idValue.equals(identifier)) { + if (idValue!=null && idValue.compareTo(identifier)==0) { String msg = messageToReplace==null?MESSAGE_TO_REPLACED_ID:messageToReplace; gco.setTextContent(msg); logger.debug("Overrided child " + idValue); + overridingRecordsMap.put(idValue, true); + //logger.debug("Returning Record IDs: "+toPrintIds.keySet()); return true; - } + } } + //logger.debug("Returning Record IDs: "+toPrintIds.keySet()); return false; } @@ -189,6 +200,70 @@ public class GetResponseRecordFilter { + /** + * Override response ids by list ids. + * + * @param getRecordsResponse the get records response + * @param idsToRemove the ids to remove + * @param messageToWrite the message to replace the right ID in the response. + * @return the input stream + * @throws IOException Signals that an I/O exception has occurred. + */ +// public static InputStream overrideResponseIdsByListIds(InputStream getRecordsResponse, List idsToRemove, String messageToWrite) throws IOException { +// +// try { +// Document doc = inputStreamToW3CDocument(getRecordsResponse); +// int overrideCount = 0; +// Map recordToOverride = new HashMap(); +// for (String identifier : idsToRemove) { +// if(overrideSummaryRecord(doc, identifier, messageToWrite, recordToOverride)) { +// overrideCount++; +// } +// } +// +// logger.info("Overrided node/s should be {}", overrideCount); +// if(logger.isDebugEnabled()) { +// for (String idRecord : recordToOverride.keySet()) { +// if(recordToOverride.get(idRecord)) { +// logger.debug("The record id {} has been overrided into response", idRecord); +// }else { +// logger.debug("No overriding for the record id: {}", idRecord); +// } +// } +// } +// +// +// //TODO IS IT POSSIBLE TO REMOVE? +// /*NodeList nodeList = doc.getElementsByTagName("csw:SearchResults"); +// if(nodeList!=null && nodeList.item(0)!=null){ +// Node nd = nodeList.item(0); +// // update staff attribute +// NamedNodeMap attr = nd.getAttributes(); +// Node numberOfRecordsMatched = attr.getNamedItem("numberOfRecordsMatched"); +// Node numberOfRecordsReturned = attr.getNamedItem("numberOfRecordsReturned"); +// logger.trace("Old numberOfRecordsMatched: "+numberOfRecordsMatched.getTextContent()); +// logger.trace("Old numberOfRecordsReturned: "+numberOfRecordsReturned.getTextContent()); +// try{ +// int oldValueM = Integer.parseInt(numberOfRecordsMatched.getTextContent()); +// int oldValueR = Integer.parseInt(numberOfRecordsReturned.getTextContent()); +// int newValueM = oldValueM-removed; +// int newValueR = oldValueR-removed; +// logger.trace("Updating numberOfRecordsMatched at: "+newValueM); +// logger.trace("Updating numberOfRecordsReturned at: "+newValueR); +// numberOfRecordsMatched.setTextContent(newValueM+""); +// numberOfRecordsReturned.setTextContent(newValueR+""); +// }catch (Exception e) { +// logger.warn("An error occurred during attribe numberOfRecordsMatched updating, skipping operation"); +// } +// }*/ +// return w3CDocumentToInputStream(doc); +// } +// catch (Exception e) { +// logger.error("An error occurred during removing IDS by List: ", e); +// return getRecordsResponse; +// } +// } + /** * Override response ids by list ids. * @@ -202,36 +277,63 @@ public class GetResponseRecordFilter { try { Document doc = inputStreamToW3CDocument(getRecordsResponse); - int override = 0; + NodeList nodes = doc.getElementsByTagName("gmd:MD_Metadata"); + logger.trace("gmd:MD_Metadata are: " + nodes.getLength()); + Map overridingRecordsMap = new HashMap(); + int overrideCount = 0; + //FOR EACH PUBLIC IDS for (String identifier : idsToRemove) { - if(overrideSummaryRecord(doc, identifier, messageToWrite)) - override++; - } - logger.info("Overrided "+override +" node/s"); + + //FOR EACH gmd:fileIdentifier RETURNED IN THE GET_RECORDS RESPONSE + //CHECKING IF IT IS A PUBLIC ID + for (int i = 0; i < nodes.getLength(); i++) { + Element mdMetadata = (Element) nodes.item(i); - //TODO IS IT POSSIBLE TO REMOVE? - /*NodeList nodeList = doc.getElementsByTagName("csw:SearchResults"); - if(nodeList!=null && nodeList.item(0)!=null){ - Node nd = nodeList.item(0); - // update staff attribute - NamedNodeMap attr = nd.getAttributes(); - Node numberOfRecordsMatched = attr.getNamedItem("numberOfRecordsMatched"); - Node numberOfRecordsReturned = attr.getNamedItem("numberOfRecordsReturned"); - logger.trace("Old numberOfRecordsMatched: "+numberOfRecordsMatched.getTextContent()); - logger.trace("Old numberOfRecordsReturned: "+numberOfRecordsReturned.getTextContent()); - try{ - int oldValueM = Integer.parseInt(numberOfRecordsMatched.getTextContent()); - int oldValueR = Integer.parseInt(numberOfRecordsReturned.getTextContent()); - int newValueM = oldValueM-removed; - int newValueR = oldValueR-removed; - logger.trace("Updating numberOfRecordsMatched at: "+newValueM); - logger.trace("Updating numberOfRecordsReturned at: "+newValueR); - numberOfRecordsMatched.setTextContent(newValueM+""); - numberOfRecordsReturned.setTextContent(newValueR+""); - }catch (Exception e) { - logger.warn("An error occurred during attribe numberOfRecordsMatched updating, skipping operation"); + // + NodeList fileIdentifierLst = mdMetadata.getElementsByTagName("gmd:fileIdentifier"); + if(fileIdentifierLst==null || fileIdentifierLst.getLength()==0 || fileIdentifierLst.item(0)==null){ + logger.info("skipping identifier: " + identifier +" it has not fileidentifier"); + break; + } + + Element id = (Element) fileIdentifierLst.item(0); + + NodeList gcoLst = id.getElementsByTagName("gco:CharacterString"); + if(gcoLst==null || gcoLst.getLength()==0 || gcoLst.item(0)==null){ + logger.debug("skipping identifier: " + identifier +" it has not gco:CharacterString"); + break; + } + + Element gco = (Element) gcoLst.item(0); + String idValue = gco.getTextContent(); + + if(overridingRecordsMap.get(idValue)==null && idValue.compareTo(messageToWrite)!=0) { + overridingRecordsMap.put(idValue, false); + } + + logger.trace("Summary gmd:fileIdentifier is: " + idValue); + if (idValue!=null && idValue.compareTo(identifier)==0) { + gco.setTextContent(messageToWrite); + logger.debug("Overrided child " + idValue); + overridingRecordsMap.put(idValue, true); + overrideCount++; + //logger.debug("Returning Record IDs: "+toPrintIds.keySet()); + break; + } } - }*/ + } + + logger.info("Overrided node/s should be {}", overrideCount); + if(logger.isDebugEnabled()) { + for (String idRecord : overridingRecordsMap.keySet()) { + if(overridingRecordsMap.get(idRecord)) { + logger.debug("The record id {} has been overrided into response", idRecord); + }else { + logger.debug("No overriding for the record id: {}", idRecord); + } + } + } + return w3CDocumentToInputStream(doc); } catch (Exception e) {