From e0d48648233ab11a06c679d04fb37c13c3ce526a Mon Sep 17 00:00:00 2001 From: Francesco Mangiacrapa Date: Fri, 9 Nov 2018 11:05:53 +0000 Subject: [PATCH] Use DataCatalogueImpl from ckan-utility-lib to discovery Ckan Portlet endpoints for scope git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/uri-resolver@174074 82a268e6-3cf1-43bd-a215-b396298e98cf --- .settings/org.eclipse.wst.common.component | 2 +- ...dingCatalogueApplicationProfilesCache.java | 238 +++++++++--------- .../caches/LoadingVREsScopeCache.java | 121 +++++++++ .../CatalogueServiceEndpointReader.java | 18 +- .../CkanCatalogueConfigurationsReader.java | 33 ++- .../resource/GetAllInfrastructureVREs.java | 213 ++++++++++++++++ .../listeners/UriResolverStartupListener.java | 6 +- .../resolver/services/CatalogueResolver.java | 36 ++- src/test/java/GetAllInfrastructureVREs.java | 204 +++++++++++++++ src/test/java/rest/TestResolvers.java | 21 +- 10 files changed, 738 insertions(+), 154 deletions(-) create mode 100644 src/main/java/org/gcube/datatransfer/resolver/caches/LoadingVREsScopeCache.java create mode 100644 src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/GetAllInfrastructureVREs.java create mode 100644 src/test/java/GetAllInfrastructureVREs.java diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 0c7a0cc..e7fefa7 100644 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -4,7 +4,7 @@ - + uses diff --git a/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingCatalogueApplicationProfilesCache.java b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingCatalogueApplicationProfilesCache.java index c6bdb76..390233e 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingCatalogueApplicationProfilesCache.java +++ b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingCatalogueApplicationProfilesCache.java @@ -1,119 +1,119 @@ -/** - * - */ -package org.gcube.datatransfer.resolver.caches; - -import java.util.concurrent.TimeUnit; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.datatransfer.resolver.catalogue.resource.ApplicationProfileReaderForCatalogueResolver; -import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; - - -/** - * The Class LoadingCatalogueApplicationProfilesCache. - * - * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) - * Nov 5, 2018 - */ -public class LoadingCatalogueApplicationProfilesCache { - - private static Logger logger = LoggerFactory.getLogger(LoadingCatalogueApplicationProfilesCache.class); - private static LoadingCache catalogueApplicationProfiles; - - static{ - - CacheLoader loader = new CacheLoader(){ - - @Override - public String load(String vreName) - throws Exception { - - logger.info("Loading the cache for vreName: "+vreName); - String fullScope = loadFullScopeByApplicationProfile(vreName); - logger.info("Returning fullScope: "+fullScope+ " for the VRE name: "+vreName); - return fullScope; - } - - }; - - RemovalListener removalListener = new RemovalListener() { - - @Override - public void onRemoval(RemovalNotification arg0) { - - logger.info("cache expired"); - //prePopulateCache(); - - } - }; - - catalogueApplicationProfiles = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite( - 1, TimeUnit.DAYS).removalListener(removalListener). - build(loader); - - - //Populating the cache at init stage - populateTheCache(); - logger.info("Pre-Loaded CatalogueApplicationProfiles cache with: "+catalogueApplicationProfiles.asMap().size()+" item/s"); - } - - - /** - * Populate the cache. - */ - private static void populateTheCache(){ - try{ - //POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER" - logger.info("Trying to pre-populate catalogue resolver cache"); - ScopeProvider.instance.set(UriResolverStartupListener.getRootContextScope()); - ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(UriResolverStartupListener.getRootContextScope(), true); - catalogueApplicationProfiles.asMap().putAll(appPrCatResolver.getHashVreNameScope()); - logger.info("Cache populated with: "+catalogueApplicationProfiles.asMap().toString()); - //logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString()); - }catch(Exception e){ - - }finally{ - - } - } - - /** - * Gets the cache. - * - * @return the cache - */ - public static LoadingCache getCache(){ - return catalogueApplicationProfiles; - } - - - - /** - * Load application profiles. - * - * @param vreName the vre name - * @return the string - */ - public static String loadFullScopeByApplicationProfile(String vreName){ - - //THIS CHECK SHOULD BE NOT NEEDED - String fullScope = catalogueApplicationProfiles.getIfPresent(vreName); - - if(fullScope==null){ - populateTheCache(); - fullScope = catalogueApplicationProfiles.getIfPresent(vreName); - } - - return fullScope; - } - -} +///** +// * +// */ +//package org.gcube.datatransfer.resolver.caches; +// +//import java.util.concurrent.TimeUnit; +// +//import org.gcube.common.scope.api.ScopeProvider; +//import org.gcube.datatransfer.resolver.catalogue.resource.ApplicationProfileReaderForCatalogueResolver; +//import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//import com.google.common.cache.CacheBuilder; +//import com.google.common.cache.CacheLoader; +//import com.google.common.cache.LoadingCache; +//import com.google.common.cache.RemovalListener; +//import com.google.common.cache.RemovalNotification; +// +// +///** +// * The Class LoadingCatalogueApplicationProfilesCache. +// * +// * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) +// * Nov 5, 2018 +// */ +//public class LoadingCatalogueApplicationProfilesCache { +// +// private static Logger logger = LoggerFactory.getLogger(LoadingCatalogueApplicationProfilesCache.class); +// private static LoadingCache catalogueApplicationProfiles; +// +// static{ +// +// CacheLoader loader = new CacheLoader(){ +// +// @Override +// public String load(String vreName) +// throws Exception { +// +// logger.info("Loading the cache for vreName: "+vreName); +// String fullScope = loadFullScopeByApplicationProfile(vreName); +// logger.info("Returning fullScope: "+fullScope+ " for the VRE name: "+vreName); +// return fullScope; +// } +// +// }; +// +// RemovalListener removalListener = new RemovalListener() { +// +// @Override +// public void onRemoval(RemovalNotification arg0) { +// +// logger.info("cache expired"); +// //prePopulateCache(); +// +// } +// }; +// +// catalogueApplicationProfiles = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite( +// 1, TimeUnit.DAYS).removalListener(removalListener). +// build(loader); +// +// +// //Populating the cache at init stage +// populateTheCache(); +// logger.info("Pre-Loaded CatalogueApplicationProfiles cache with: "+catalogueApplicationProfiles.asMap().size()+" item/s"); +// } +// +// +// /** +// * Populate the cache. +// */ +// private static void populateTheCache(){ +// try{ +// //POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER" +// logger.info("Trying to pre-populate catalogue resolver cache"); +// ScopeProvider.instance.set(UriResolverStartupListener.getRootContextScope()); +// ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(UriResolverStartupListener.getRootContextScope(), true); +// catalogueApplicationProfiles.asMap().putAll(appPrCatResolver.getHashVreNameScope()); +// logger.info("Cache populated with: "+catalogueApplicationProfiles.asMap().toString()); +// //logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString()); +// }catch(Exception e){ +// +// }finally{ +// +// } +// } +// +// /** +// * Gets the cache. +// * +// * @return the cache +// */ +// public static LoadingCache getCache(){ +// return catalogueApplicationProfiles; +// } +// +// +// +// /** +// * Load application profiles. +// * +// * @param vreName the vre name +// * @return the string +// */ +// public static String loadFullScopeByApplicationProfile(String vreName){ +// +// //THIS CHECK SHOULD BE NOT NEEDED +// String fullScope = catalogueApplicationProfiles.getIfPresent(vreName); +// +// if(fullScope==null){ +// populateTheCache(); +// fullScope = catalogueApplicationProfiles.getIfPresent(vreName); +// } +// +// return fullScope; +// } +// +//} diff --git a/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingVREsScopeCache.java b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingVREsScopeCache.java new file mode 100644 index 0000000..33c638b --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/caches/LoadingVREsScopeCache.java @@ -0,0 +1,121 @@ +/** + * + */ +package org.gcube.datatransfer.resolver.caches; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.datatransfer.resolver.catalogue.resource.GetAllInfrastructureVREs; +import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; + + + +/** + * The Class LoadingVREsScopeCache. + * + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * Nov 8, 2018 + */ +public class LoadingVREsScopeCache { + + private static Logger logger = LoggerFactory.getLogger(LoadingVREsScopeCache.class); + private static LoadingCache vresNameToScope; + + static{ + + CacheLoader loader = new CacheLoader(){ + + @Override + public String load(String vreName) + throws Exception { + + logger.info("Loading the cache for vreName: "+vreName); + String fullScope = loadFullScopeforVreName(vreName); + logger.info("Returning fullScope: "+fullScope+ " for the VRE name: "+vreName); + return fullScope; + } + + }; + + RemovalListener removalListener = new RemovalListener() { + + @Override + public void onRemoval(RemovalNotification arg0) { + + logger.info("cache expired"); + //prePopulateCache(); + + } + }; + + vresNameToScope = CacheBuilder.newBuilder().maximumSize(300).expireAfterWrite( + 1, TimeUnit.DAYS).removalListener(removalListener). + build(loader); + + + //Populating the cache at init stage + populateTheCache(); + logger.info("Pre-Loaded VRE to Scope cache with: "+vresNameToScope.asMap().size()+" item/s"); + } + + + /** + * Populate the cache. + */ + private static void populateTheCache(){ + try{ + //POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER" + logger.info("Trying to pre-populate VRE Names to Scope cache"); + ScopeProvider.instance.set(UriResolverStartupListener.getRootContextScope()); + Map map = GetAllInfrastructureVREs.loadMapOFVreNameToScope(UriResolverStartupListener.getRootContextScope()); + vresNameToScope.asMap().putAll(map); + logger.info("Cache populated with: "+vresNameToScope.asMap().toString()); + //logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString()); + }catch(Exception e){ + + }finally{ + + } + } + + /** + * Gets the cache. + * + * @return the cache + */ + public static LoadingCache getCache(){ + return vresNameToScope; + } + + + + /** + * Load application profiles. + * + * @param vreName the vre name + * @return the string + */ + public static String loadFullScopeforVreName(String vreName){ + + //THIS CHECK SHOULD BE NOT NEEDED + String fullScope = vresNameToScope.getIfPresent(vreName); + + if(fullScope==null){ + populateTheCache(); + fullScope = vresNameToScope.getIfPresent(vreName); + } + + return fullScope; + } + +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/catalogue/endpoint/CatalogueServiceEndpointReader.java b/src/main/java/org/gcube/datatransfer/resolver/catalogue/endpoint/CatalogueServiceEndpointReader.java index 03414bb..9b81eb2 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/catalogue/endpoint/CatalogueServiceEndpointReader.java +++ b/src/main/java/org/gcube/datatransfer/resolver/catalogue/endpoint/CatalogueServiceEndpointReader.java @@ -84,7 +84,6 @@ public class CatalogueServiceEndpointReader { } - /** * Gets the catalogue url. * @@ -92,7 +91,7 @@ public class CatalogueServiceEndpointReader { */ public static String getCatalogueUrl() { String scope = ScopeProvider.instance.get(); - logger.debug("Getting Catalogue URL for scope: "+scope +" read from ScopeProvider"); + logger.debug("Getting Catalogue URL for scope: "+scope +" read from CacheCkanDataCatalogue"); String catalogueURLForScope = cacheCkanDataCatalogue.get(scope); if(catalogueURLForScope==null){ @@ -117,6 +116,21 @@ public class CatalogueServiceEndpointReader { } + + /** + * Gets the data catalogue impl. + * + * @return the data catalogue impl + * @throws Exception + */ + public static DataCatalogueImpl getDataCatalogueImpl() throws Exception { + String scope = ScopeProvider.instance.get(); + return new DataCatalogueImpl(scope); + } + + + + // /** // * The main method. // * diff --git a/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/CkanCatalogueConfigurationsReader.java b/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/CkanCatalogueConfigurationsReader.java index 3550386..b7473ca 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/CkanCatalogueConfigurationsReader.java +++ b/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/CkanCatalogueConfigurationsReader.java @@ -20,6 +20,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.gcube.common.resources.gcore.utils.XPathHelper; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl; import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileNotFoundException; import org.gcube.datatransfer.resolver.catalogue.endpoint.CatalogueServiceEndpointReader; import org.gcube.resources.discovery.client.api.DiscoveryClient; @@ -59,6 +60,33 @@ public class CkanCatalogueConfigurationsReader { GatewayCKANCatalogueReference links = new GatewayCKANCatalogueReference(); links.setScope(ScopeProvider.instance.get()); + DataCatalogueImpl catalogueImpl = CatalogueServiceEndpointReader.getDataCatalogueImpl(); + + String privatePortletURL = catalogueImpl.getPortletUrl(); + links.setPrivatePortletURL(privatePortletURL); + + //Building public URL from private portlet URL + try{ + URI toURL = new URI(privatePortletURL); + String publicURL = privatePortletURL.startsWith("https://")?"https://"+toURL.getHost():"http://"+toURL.getHost(); + String realiveURLToPublicCtlg = getRelativeURLToCatalogue(); + links.setPublicPortletURL(publicURL+"/"+realiveURLToPublicCtlg); + }catch(Exception e){ + logger.warn("Erron on generating public catalogue URL from private URL: "+privatePortletURL, e); + } + + //Getting the CKAN Portlet URL for current scope + try{ + String ckanPortletURL = catalogueImpl.getCatalogueUrl(); + links.setCkanURL(ckanPortletURL); + }catch(Exception e){ + logger.warn("Erron on getting CKAN Porlet URL for scope: "+ScopeProvider.instance.get(), e); + } + + return links; + + + /* String privatePortletURL = getPortletUrlForScopeFromIS(); links.setPrivatePortletURL(privatePortletURL); @@ -81,6 +109,9 @@ public class CkanCatalogueConfigurationsReader { } return links; + */ + + } @@ -113,7 +144,7 @@ public class CkanCatalogueConfigurationsReader { * @return the portlet url for scope from is * @throws Exception the exception */ - private static String getPortletUrlForScopeFromIS() throws Exception { + protected static String getPortletUrlForScopeFromIS() throws Exception { String scope = ScopeProvider.instance.get(); logger.debug("Trying to fetch applicationProfile profile from the infrastructure for " + diff --git a/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/GetAllInfrastructureVREs.java b/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/GetAllInfrastructureVREs.java new file mode 100644 index 0000000..9b0c76d --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/GetAllInfrastructureVREs.java @@ -0,0 +1,213 @@ +package org.gcube.datatransfer.resolver.catalogue.resource; +import static org.gcube.resources.discovery.icclient.ICFactory.client; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.gcube.common.resources.gcore.utils.XPathHelper; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.Query; +import org.gcube.resources.discovery.client.queries.impl.QueryBox; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; + + +/** + * The Class GetAllInfrastructureVREs. + * + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * Nov 8, 2018 + */ +/** + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Nov 8, 2018 + */ +public class GetAllInfrastructureVREs { + + public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureVREs.class); + + protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()"; + + /** + * Load map of vre name to scope. + * + * @param rootScope the root scope + * @return the map of binding between (VRE_NAME, FULL_SCOPE_OF_VRE_NAME) + * @throws Exception + */ + public static Map loadMapOFVreNameToScope(String rootScope) throws Exception{ + + String originalScope = ScopeProvider.instance.get(); + + try{ + + ScopeProvider.instance.set(rootScope); + String secondaryType = "INFRASTRUCTURE"; + List listVOScopes = getListOfVOScopes(secondaryType); + Map vreNameFullScope = new HashMap(); + logger.info("Searching for secondaryType="+secondaryType +" scope/s found is/are: " +listVOScopes); + + secondaryType = "VRE"; + //int noVOTypeCount = 0; + for (String voScope : listVOScopes) { + int count = voScope.length() - voScope.replace("/", "").length(); + //IS A VO + if(count==2){ + logger.info(voScope +" is a VO..."); + ScopeProvider.instance.set(voScope); + List listVREs = getListOfResourcesForSecondaryType(secondaryType); + logger.debug("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs); + for (String vreName : listVREs) { + String vreScope = String.format("%s/%s", voScope,vreName); + vreNameFullScope.put(vreName, vreScope); + } + + }else{ + //noVOTypeCount++; + logger.info(voScope +" is not a VO, skipping it"); + } + } + + /*System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount)); + for (String vreName : vreNameFullScope.keySet()) { + System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName)); + }*/ + + logger.info("Total VRE is: "+vreNameFullScope.size()); + return vreNameFullScope; + + }catch(Exception e ){ + throw new Exception("Error on loading the map of VRE nameto scope: ", e); + } + finally{ + if(originalScope!=null && !originalScope.isEmpty()){ + ScopeProvider.instance.set(originalScope); + }else + ScopeProvider.instance.reset(); + } + } + + + /** + * Gets the list of resources for secondary type. + * + * @param secondaryType the secondary type + * @return the list of resource names for the input secondary type + */ + private static List getListOfResourcesForSecondaryType(String secondaryType) { + + String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile"; + + List listResourceName = new ArrayList(); + + try { + logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType); + Query q = new QueryBox(queryString); + DiscoveryClient client = client(); + List listGenericResources = client.submit(q); + + logger.info("# of GenericResource returned are: "+listGenericResources.size()); + + if (listGenericResources == null || listGenericResources.size() == 0) + throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get()); + else { + + + for (String genericResource : listGenericResources) { + try{ + String elem = genericResource; + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = docBuilder.parse(new InputSource(new StringReader(elem))); + Element rootElement = document.getDocumentElement(); + XPathHelper helper = new XPathHelper(rootElement); + List resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT); + + if(resourceNames!=null && resourceNames.size()>0) + listResourceName.add(resourceNames.get(0)); + + }catch(Exception e){ + throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get()); + } + } + + } + + } catch (Exception e) { + logger.error("Error while trying to fetch generic resource from the infrastructure", e); + } + + return listResourceName; + + } + + + /** + * Gets the list of resources for secondary type. + * + * @param secondaryType the secondary type + * @return the list of resource names for the input secondary type + */ + private static List getListOfVOScopes(String secondaryType) { + + String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile"; + + List listOfVOScopes = new ArrayList(); + + try { + logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType); + Query q = new QueryBox(queryString); + DiscoveryClient client = client(); + List listGenericResources = client.submit(q); + + logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size()); + + if (listGenericResources == null || listGenericResources.size() == 0) + throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get()); + else { + + + for (String genericResource : listGenericResources) { + try{ + String elem = genericResource; + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = docBuilder.parse(new InputSource(new StringReader(elem))); + Element rootElement = document.getDocumentElement(); + XPathHelper helper = new XPathHelper(rootElement); +// List resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT); +// +// if(resourceNames!=null && resourceNames.size()>0) +// listResourceName.add(resourceNames.get(0)); + + List scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()"); + for (String scopeFound : scopes) { + listOfVOScopes.add(scopeFound); + } + + }catch(Exception e){ + throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get()); + } + } + + } + + } catch (Exception e) { + logger.error("Error while trying to fetch generic resource from the infrastructure", e); + } + + return listOfVOScopes; + + } +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/listeners/UriResolverStartupListener.java b/src/main/java/org/gcube/datatransfer/resolver/listeners/UriResolverStartupListener.java index a7f49c9..9955510 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/listeners/UriResolverStartupListener.java +++ b/src/main/java/org/gcube/datatransfer/resolver/listeners/UriResolverStartupListener.java @@ -15,9 +15,10 @@ import javax.servlet.ServletContextListener; import javax.servlet.ServletException; import javax.servlet.annotation.WebListener; -import org.gcube.datatransfer.resolver.caches.LoadingCatalogueApplicationProfilesCache; +//import org.gcube.datatransfer.resolver.caches.LoadingCatalogueApplicationProfilesCache; import org.gcube.datatransfer.resolver.caches.LoadingGeonetworkInstanceCache; import org.gcube.datatransfer.resolver.caches.LoadingGisViewerApplicationURLCache; +import org.gcube.datatransfer.resolver.caches.LoadingVREsScopeCache; import org.gcube.datatransfer.resolver.gis.property.ApplicationProfilePropertyReader; import org.gcube.datatransfer.resolver.gis.property.PropertyFileNotFoundException; import org.gcube.smartgears.ContextProvider; @@ -79,7 +80,8 @@ public class UriResolverStartupListener implements ServletContextListener { //init the caches new LoadingGeonetworkInstanceCache(); new LoadingGisViewerApplicationURLCache(); - new LoadingCatalogueApplicationProfilesCache(); + new LoadingVREsScopeCache(); + //new LoadingCatalogueApplicationProfilesCache(); logger.info("Context initialized with: "); logger.info("Scope: "+rootContextScope); diff --git a/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java b/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java index 5d13d4b..dbd4ec6 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java +++ b/src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java @@ -1,6 +1,7 @@ package org.gcube.datatransfer.resolver.services; import java.net.URL; +import java.util.concurrent.ExecutionException; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; @@ -17,14 +18,12 @@ import javax.ws.rs.core.Response.Status; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.datatransfer.resolver.ResourceCatalogueCodes; -import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileNotFoundException; -import org.gcube.datatransfer.resolver.caches.LoadingCatalogueApplicationProfilesCache; +import org.gcube.datatransfer.resolver.caches.LoadingVREsScopeCache; import org.gcube.datatransfer.resolver.catalogue.CatalogueRequest; -import org.gcube.datatransfer.resolver.catalogue.resource.ApplicationProfileReaderForCatalogueResolver; import org.gcube.datatransfer.resolver.catalogue.resource.CkanCatalogueConfigurationsReader; import org.gcube.datatransfer.resolver.catalogue.resource.GatewayCKANCatalogueReference; -import org.gcube.datatransfer.resolver.catalogue.resource.UpdateApplicationProfileCatalogueResolver; -import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener; +import org.gcube.datatransfer.resolver.catalogue.resource.GetAllInfrastructureVREs; +import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.smartgears.utils.InnerMethodName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +34,7 @@ import eu.trentorise.opendata.jackan.model.CkanDataset; public class CatalogueResolver { private static Logger logger = LoggerFactory.getLogger(CatalogueResolver.class); + private static String helpURI = "https://wiki.gcube-system.org/gcube/URI_Resolver#CATALOGUE_Resolver"; /** The scope to enc decr. */ //private String scopeToEncDecr = null; @@ -51,14 +51,16 @@ public class CatalogueResolver { //ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(vreName, true); - String fullScope = LoadingCatalogueApplicationProfilesCache.getCache().get(vreName); - logger.debug("Read fullScope: "+fullScope + " for VRE_NAME: "+vreName +" into Application Profile "+ApplicationProfileReaderForCatalogueResolver.RESOURCE_NAME); + String fullScope = LoadingVREsScopeCache.getCache().get(vreName); + logger.info("Read fullScope: "+fullScope + " for VRE_NAME: "+vreName +" from cache created by: "+GetAllInfrastructureVREs.class.getSimpleName()); //String fullScope = appPrCatResolver.getHashVreNameScope().get(vreName); ScopeProvider.instance.set(fullScope); GatewayCKANCatalogueReference ckanCatalogueReference = CkanCatalogueConfigurationsReader.loadCatalogueEndPoints(); + logger.info("For scope "+fullScope+" loaded end points: "+ckanCatalogueReference); + //IS THE PRODUCT PLUBLIC OR PRIVATE? //USING ACCESS TO PUBLIC PORTLET IF THE ITEM IS PUBLIC, OTHERWISE ACCESS TO PRIVATE PORTLET String ckanPorltetUrl = ckanCatalogueReference.getPrivatePortletURL(); @@ -107,20 +109,32 @@ public class CatalogueResolver { String serverUrl = getServerURL(req); final String vreName = scope.substring(scope.lastIndexOf("/")+1, scope.length()); + String fullScope = null; + //CHECK IF THE vreName has a valid scope, so it is a valid VRE + try { + fullScope = LoadingVREsScopeCache.getCache().get(vreName); + } + catch (ExecutionException e1) { + logger.error("Error on getting full scope for vre name: "+vreName, e1); + } + + if(fullScope==null) + ExceptionManager.throwWrongParameterException(req, "The scope '"+scope+"' does not matching any scope in the infrastructure. Is it valid?", this.getClass(), helpURI); ResourceCatalogueCodes rc = ResourceCatalogueCodes.valueOfCodeValue(jsonRequest.getEntity_context()); if(rc==null){ logger.error("Entity context is null/malformed"); - throw new WebApplicationException("Entity context is null/malformed", Status.BAD_REQUEST); + ExceptionManager.throwBadRequestException(req, "Entity context is null/malformed", this.getClass(), helpURI); + //throw new WebApplicationException("Entity context is null/malformed", Status.BAD_REQUEST); } String linkURL = String.format("%s/%s/%s/%s", serverUrl, rc.getId(), vreName, jsonRequest.getEntity_name()); - logger.info("Writing Decoded Catalogue Link: "+linkURL); + logger.info("Returining Catalogue URL: "+linkURL); //IT'S GOING TO UPDATE THE GENERIC RESOURCE IF IS NEEDED - final String fullscope = scope; + /*final String fullscope = scope; new Thread(){ public void run() { try { @@ -133,7 +147,7 @@ public class CatalogueResolver { logger.error("Error during validating Application Profile", e); } }; - }.start(); + }.start();*/ return Response.ok(linkURL).build(); diff --git a/src/test/java/GetAllInfrastructureVREs.java b/src/test/java/GetAllInfrastructureVREs.java new file mode 100644 index 0000000..96f800f --- /dev/null +++ b/src/test/java/GetAllInfrastructureVREs.java @@ -0,0 +1,204 @@ +import static org.gcube.resources.discovery.icclient.ICFactory.client; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.gcube.common.resources.gcore.utils.XPathHelper; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.Query; +import org.gcube.resources.discovery.client.queries.impl.QueryBox; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; + + +/** + * The Class GetAllInfrastructureVREs. + * + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * Nov 8, 2018 + */ +/** + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Nov 8, 2018 + */ +public class GetAllInfrastructureVREs { + + public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureVREs.class); + + protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()"; + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) { + + + //TODO TOKEN TO ROOT SCOPE + String rootScope = "/gcube"; + //String rootScope = "/d4science.research-infrastructures.eu"; + ScopeProvider.instance.set(rootScope); + + String secondaryType = "INFRASTRUCTURE"; + List listVOScopes = getListOfVOScopes("INFRASTRUCTURE"); + + System.out.println("Searching for secondaryType="+secondaryType +" scope/s found is/are: " +listVOScopes); + + Map vreNameFullScope = new HashMap(); + + secondaryType = "VRE"; + int noVOTypeCount = 0; + for (String voScope : listVOScopes) { + int count = voScope.length() - voScope.replace("/", "").length(); + //IS A VO + if(count==2){ + logger.info(voScope +" is a VO..."); + ScopeProvider.instance.set(voScope); + List listVREs = getListOfResourcesForSecondaryType(secondaryType); + System.out.println("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs); + for (String vreName : listVREs) { + String vreScope = String.format("%s/%s", voScope,vreName); + vreNameFullScope.put(vreName, vreScope); + } + + }else{ + noVOTypeCount++; + System.out.println(voScope +" is not a VO, skipping it"); + } + } + + + System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount)); + for (String vreName : vreNameFullScope.keySet()) { + System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName)); + } + + System.out.println("Total VRE is: "+vreNameFullScope.size()); + + } + + + /** + * Gets the list of resources for secondary type. + * + * @param secondaryType the secondary type + * @return the list of resource names for the input secondary type + */ + protected static List getListOfResourcesForSecondaryType(String secondaryType) { + + String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile"; + + List listResourceName = new ArrayList(); + + try { + logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType); + Query q = new QueryBox(queryString); + DiscoveryClient client = client(); + List listGenericResources = client.submit(q); + + logger.info("# of GenericResource returned are: "+listGenericResources.size()); + + if (listGenericResources == null || listGenericResources.size() == 0) + throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get()); + else { + + + for (String genericResource : listGenericResources) { + try{ + String elem = genericResource; + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = docBuilder.parse(new InputSource(new StringReader(elem))); + Element rootElement = document.getDocumentElement(); + XPathHelper helper = new XPathHelper(rootElement); + List resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT); + + if(resourceNames!=null && resourceNames.size()>0) + listResourceName.add(resourceNames.get(0)); + + }catch(Exception e){ + throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get()); + } + } + + } + + } catch (Exception e) { + logger.error("Error while trying to fetch generic resource from the infrastructure", e); + } + + return listResourceName; + + } + + + /** + * Gets the list of resources for secondary type. + * + * @param secondaryType the secondary type + * @return the list of resource names for the input secondary type + */ + protected static List getListOfVOScopes(String secondaryType) { + + String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile"; + + List listOfVOScopes = new ArrayList(); + + try { + logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType); + Query q = new QueryBox(queryString); + DiscoveryClient client = client(); + List listGenericResources = client.submit(q); + + logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size()); + + if (listGenericResources == null || listGenericResources.size() == 0) + throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get()); + else { + + + for (String genericResource : listGenericResources) { + try{ + String elem = genericResource; + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = docBuilder.parse(new InputSource(new StringReader(elem))); + Element rootElement = document.getDocumentElement(); + XPathHelper helper = new XPathHelper(rootElement); +// List resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT); +// +// if(resourceNames!=null && resourceNames.size()>0) +// listResourceName.add(resourceNames.get(0)); + + List scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()"); + for (String scopeFound : scopes) { + listOfVOScopes.add(scopeFound); + } + + }catch(Exception e){ + throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get()); + } + } + + } + + } catch (Exception e) { + logger.error("Error while trying to fetch generic resource from the infrastructure", e); + } + + return listOfVOScopes; + + } +} diff --git a/src/test/java/rest/TestResolvers.java b/src/test/java/rest/TestResolvers.java index 3623f63..01d9c7f 100644 --- a/src/test/java/rest/TestResolvers.java +++ b/src/test/java/rest/TestResolvers.java @@ -254,8 +254,8 @@ public class TestResolvers { * * @throws Exception the exception */ - //@Test - public void testCatalogueResolverCreatePubrivatetemURL() throws Exception{ + @Test + public void testCatalogueResolverCreatePubrivatItemURL() throws Exception{ String entityName = "dynamic_reporting"; //String entityContext = "product"; @@ -302,7 +302,7 @@ public class TestResolvers { * * @throws Exception the exception */ - @Test + //@Test public void testGisResolver() throws Exception{ String gisUUID = "55c19a1f-214b-4f81-9220-fba09fcfa91f"; @@ -341,21 +341,6 @@ public class TestResolvers { } - - - - - - - - - - - - - - - /** * Gets the content reponse. *