gCubeSystem
/
uri-resolver
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use DataCatalogueImpl from ckan-utility-lib to discovery Ckan Portlet endpoints for scope 

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/uri-resolver@174074 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Francesco Mangiacrapa 2018-11-09 11:05:53 +00:00
parent 2b7712600a
commit e0d4864823
10 changed files with 738 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.settings/org.eclipse.wst.common.component
View File

@ -4,7 +4,7 @@
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/> <wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/> <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/> <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
<dependent-module archiveName="ckan-util-library-2.7.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/ckan-util-library-TRUNK/ckan-util-library-TRUNK"> <dependent-module archiveName="ckan-util-library-2.8.0-SNAPSHOT.jar" deploy-path="/WEB-INF/lib" handle="module:/resource/ckan-util-library-TRUNK/ckan-util-library-TRUNK">
<dependency-type>uses</dependency-type> <dependency-type>uses</dependency-type>
</dependent-module> </dependent-module>
<property name="context-root" value="uri-resolver"/> <property name="context-root" value="uri-resolver"/>

238
src/main/java/org/gcube/datatransfer/resolver/caches/LoadingCatalogueApplicationProfilesCache.java
View File

@ -1,119 +1,119 @@
/** ///**
* // *
*/ // */
package org.gcube.datatransfer.resolver.caches; //package org.gcube.datatransfer.resolver.caches;
//
import java.util.concurrent.TimeUnit; //import java.util.concurrent.TimeUnit;
//
import org.gcube.common.scope.api.ScopeProvider; //import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datatransfer.resolver.catalogue.resource.ApplicationProfileReaderForCatalogueResolver; //import org.gcube.datatransfer.resolver.catalogue.resource.ApplicationProfileReaderForCatalogueResolver;
import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener; //import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener;
import org.slf4j.Logger; //import org.slf4j.Logger;
import org.slf4j.LoggerFactory; //import org.slf4j.LoggerFactory;
//
import com.google.common.cache.CacheBuilder; //import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader; //import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache; //import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener; //import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification; //import com.google.common.cache.RemovalNotification;
//
//
/** ///**
* The Class LoadingCatalogueApplicationProfilesCache. // * The Class LoadingCatalogueApplicationProfilesCache.
* // *
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) // * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
* Nov 5, 2018 // * Nov 5, 2018
*/ // */
public class LoadingCatalogueApplicationProfilesCache { //public class LoadingCatalogueApplicationProfilesCache {
//
private static Logger logger = LoggerFactory.getLogger(LoadingCatalogueApplicationProfilesCache.class); // private static Logger logger = LoggerFactory.getLogger(LoadingCatalogueApplicationProfilesCache.class);
private static LoadingCache<String, String> catalogueApplicationProfiles; // private static LoadingCache<String, String> catalogueApplicationProfiles;
//
static{ // static{
//
CacheLoader<String, String> loader = new CacheLoader<String, String>(){ // CacheLoader<String, String> loader = new CacheLoader<String, String>(){
//
@Override // @Override
public String load(String vreName) // public String load(String vreName)
throws Exception { // throws Exception {
//
logger.info("Loading the cache for vreName: "+vreName); // logger.info("Loading the cache for vreName: "+vreName);
String fullScope = loadFullScopeByApplicationProfile(vreName); // String fullScope = loadFullScopeByApplicationProfile(vreName);
logger.info("Returning fullScope: "+fullScope+ " for the VRE name: "+vreName); // logger.info("Returning fullScope: "+fullScope+ " for the VRE name: "+vreName);
return fullScope; // return fullScope;
} // }
//
}; // };
//
RemovalListener<String, String> removalListener = new RemovalListener<String, String>() { // RemovalListener<String, String> removalListener = new RemovalListener<String, String>() {
//
@Override // @Override
public void onRemoval(RemovalNotification<String, String> arg0) { // public void onRemoval(RemovalNotification<String, String> arg0) {
//
logger.info("cache expired"); // logger.info("cache expired");
//prePopulateCache(); // //prePopulateCache();
//
} // }
}; // };
//
catalogueApplicationProfiles = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite( // catalogueApplicationProfiles = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(
1, TimeUnit.DAYS).removalListener(removalListener). // 1, TimeUnit.DAYS).removalListener(removalListener).
build(loader); // build(loader);
//
//
//Populating the cache at init stage // //Populating the cache at init stage
populateTheCache(); // populateTheCache();
logger.info("Pre-Loaded CatalogueApplicationProfiles cache with: "+catalogueApplicationProfiles.asMap().size()+" item/s"); // logger.info("Pre-Loaded CatalogueApplicationProfiles cache with: "+catalogueApplicationProfiles.asMap().size()+" item/s");
} // }
//
//
/** // /**
* Populate the cache. // * Populate the cache.
*/ // */
private static void populateTheCache(){ // private static void populateTheCache(){
try{ // try{
//POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER" // //POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER"
logger.info("Trying to pre-populate catalogue resolver cache"); // logger.info("Trying to pre-populate catalogue resolver cache");
ScopeProvider.instance.set(UriResolverStartupListener.getRootContextScope()); // ScopeProvider.instance.set(UriResolverStartupListener.getRootContextScope());
ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(UriResolverStartupListener.getRootContextScope(), true); // ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(UriResolverStartupListener.getRootContextScope(), true);
catalogueApplicationProfiles.asMap().putAll(appPrCatResolver.getHashVreNameScope()); // catalogueApplicationProfiles.asMap().putAll(appPrCatResolver.getHashVreNameScope());
logger.info("Cache populated with: "+catalogueApplicationProfiles.asMap().toString()); // logger.info("Cache populated with: "+catalogueApplicationProfiles.asMap().toString());
//logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString()); // //logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString());
}catch(Exception e){ // }catch(Exception e){
//
}finally{ // }finally{
//
} // }
} // }
//
/** // /**
* Gets the cache. // * Gets the cache.
* // *
* @return the cache // * @return the cache
*/ // */
public static LoadingCache<String, String> getCache(){ // public static LoadingCache<String, String> getCache(){
return catalogueApplicationProfiles; // return catalogueApplicationProfiles;
} // }
//
//
//
/** // /**
* Load application profiles. // * Load application profiles.
* // *
* @param vreName the vre name // * @param vreName the vre name
* @return the string // * @return the string
*/ // */
public static String loadFullScopeByApplicationProfile(String vreName){ // public static String loadFullScopeByApplicationProfile(String vreName){
//
//THIS CHECK SHOULD BE NOT NEEDED // //THIS CHECK SHOULD BE NOT NEEDED
String fullScope = catalogueApplicationProfiles.getIfPresent(vreName); // String fullScope = catalogueApplicationProfiles.getIfPresent(vreName);
//
if(fullScope==null){ // if(fullScope==null){
populateTheCache(); // populateTheCache();
fullScope = catalogueApplicationProfiles.getIfPresent(vreName); // fullScope = catalogueApplicationProfiles.getIfPresent(vreName);
} // }
//
return fullScope; // return fullScope;
} // }
//
} //}

121
src/main/java/org/gcube/datatransfer/resolver/caches/LoadingVREsScopeCache.java Normal file
View File

@ -0,0 +1,121 @@
/**
*
*/
package org.gcube.datatransfer.resolver.caches;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datatransfer.resolver.catalogue.resource.GetAllInfrastructureVREs;
import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
/**
* The Class LoadingVREsScopeCache.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
* Nov 8, 2018
*/
public class LoadingVREsScopeCache {
private static Logger logger = LoggerFactory.getLogger(LoadingVREsScopeCache.class);
private static LoadingCache<String, String> vresNameToScope;
static{
CacheLoader<String, String> loader = new CacheLoader<String, String>(){
@Override
public String load(String vreName)
throws Exception {
logger.info("Loading the cache for vreName: "+vreName);
String fullScope = loadFullScopeforVreName(vreName);
logger.info("Returning fullScope: "+fullScope+ " for the VRE name: "+vreName);
return fullScope;
}
};
RemovalListener<String, String> removalListener = new RemovalListener<String, String>() {
@Override
public void onRemoval(RemovalNotification<String, String> arg0) {
logger.info("cache expired");
//prePopulateCache();
}
};
vresNameToScope = CacheBuilder.newBuilder().maximumSize(300).expireAfterWrite(
1, TimeUnit.DAYS).removalListener(removalListener).
build(loader);
//Populating the cache at init stage
populateTheCache();
logger.info("Pre-Loaded VRE to Scope cache with: "+vresNameToScope.asMap().size()+" item/s");
}
/**
* Populate the cache.
*/
private static void populateTheCache(){
try{
//POPULATE THE CACHE READING THE RESOURCE "CATALOGUE-RESOLVER"
logger.info("Trying to pre-populate VRE Names to Scope cache");
ScopeProvider.instance.set(UriResolverStartupListener.getRootContextScope());
Map<String, String> map = GetAllInfrastructureVREs.loadMapOFVreNameToScope(UriResolverStartupListener.getRootContextScope());
vresNameToScope.asMap().putAll(map);
logger.info("Cache populated with: "+vresNameToScope.asMap().toString());
//logger.info("Pre-Loaded CatalogueApplicationProfiles cache is: "+catalogueApplicationProfiles.asMap().toString());
}catch(Exception e){
}finally{
}
}
/**
* Gets the cache.
*
* @return the cache
*/
public static LoadingCache<String, String> getCache(){
return vresNameToScope;
}
/**
* Load application profiles.
*
* @param vreName the vre name
* @return the string
*/
public static String loadFullScopeforVreName(String vreName){
//THIS CHECK SHOULD BE NOT NEEDED
String fullScope = vresNameToScope.getIfPresent(vreName);
if(fullScope==null){
populateTheCache();
fullScope = vresNameToScope.getIfPresent(vreName);
}
return fullScope;
}
}

18
src/main/java/org/gcube/datatransfer/resolver/catalogue/endpoint/CatalogueServiceEndpointReader.java
View File

@ -84,7 +84,6 @@ public class CatalogueServiceEndpointReader {
} }
/** /**
* Gets the catalogue url. * Gets the catalogue url.
* *
@ -92,7 +91,7 @@ public class CatalogueServiceEndpointReader {
*/ */
public static String getCatalogueUrl() { public static String getCatalogueUrl() {
String scope = ScopeProvider.instance.get(); String scope = ScopeProvider.instance.get();
logger.debug("Getting Catalogue URL for scope: "+scope +" read from ScopeProvider"); logger.debug("Getting Catalogue URL for scope: "+scope +" read from CacheCkanDataCatalogue");
String catalogueURLForScope = cacheCkanDataCatalogue.get(scope); String catalogueURLForScope = cacheCkanDataCatalogue.get(scope);
if(catalogueURLForScope==null){ if(catalogueURLForScope==null){
@ -117,6 +116,21 @@ public class CatalogueServiceEndpointReader {
} }
/**
* Gets the data catalogue impl.
*
* @return the data catalogue impl
* @throws Exception
*/
public static DataCatalogueImpl getDataCatalogueImpl() throws Exception {
String scope = ScopeProvider.instance.get();
return new DataCatalogueImpl(scope);
}
// /** // /**
// * The main method. // * The main method.
// * // *

33
src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/CkanCatalogueConfigurationsReader.java
View File

@ -20,6 +20,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import org.gcube.common.resources.gcore.utils.XPathHelper; import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datacatalogue.ckanutillibrary.server.DataCatalogueImpl;
import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileNotFoundException; import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileNotFoundException;
import org.gcube.datatransfer.resolver.catalogue.endpoint.CatalogueServiceEndpointReader; import org.gcube.datatransfer.resolver.catalogue.endpoint.CatalogueServiceEndpointReader;
import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.api.DiscoveryClient;
@ -59,6 +60,33 @@ public class CkanCatalogueConfigurationsReader {
GatewayCKANCatalogueReference links = new GatewayCKANCatalogueReference(); GatewayCKANCatalogueReference links = new GatewayCKANCatalogueReference();
links.setScope(ScopeProvider.instance.get()); links.setScope(ScopeProvider.instance.get());
DataCatalogueImpl catalogueImpl = CatalogueServiceEndpointReader.getDataCatalogueImpl();
String privatePortletURL = catalogueImpl.getPortletUrl();
links.setPrivatePortletURL(privatePortletURL);
//Building public URL from private portlet URL
try{
URI toURL = new URI(privatePortletURL);
String publicURL = privatePortletURL.startsWith("https://")?"https://"+toURL.getHost():"http://"+toURL.getHost();
String realiveURLToPublicCtlg = getRelativeURLToCatalogue();
links.setPublicPortletURL(publicURL+"/"+realiveURLToPublicCtlg);
}catch(Exception e){
logger.warn("Erron on generating public catalogue URL from private URL: "+privatePortletURL, e);
}
//Getting the CKAN Portlet URL for current scope
try{
String ckanPortletURL = catalogueImpl.getCatalogueUrl();
links.setCkanURL(ckanPortletURL);
}catch(Exception e){
logger.warn("Erron on getting CKAN Porlet URL for scope: "+ScopeProvider.instance.get(), e);
}
return links;
/*
String privatePortletURL = getPortletUrlForScopeFromIS(); String privatePortletURL = getPortletUrlForScopeFromIS();
links.setPrivatePortletURL(privatePortletURL); links.setPrivatePortletURL(privatePortletURL);
@ -81,6 +109,9 @@ public class CkanCatalogueConfigurationsReader {
} }
return links; return links;
*/
} }
@ -113,7 +144,7 @@ public class CkanCatalogueConfigurationsReader {
* @return the portlet url for scope from is * @return the portlet url for scope from is
* @throws Exception the exception * @throws Exception the exception
*/ */
private static String getPortletUrlForScopeFromIS() throws Exception { protected static String getPortletUrlForScopeFromIS() throws Exception {
String scope = ScopeProvider.instance.get(); String scope = ScopeProvider.instance.get();
logger.debug("Trying to fetch applicationProfile profile from the infrastructure for " + logger.debug("Trying to fetch applicationProfile profile from the infrastructure for " +

213
src/main/java/org/gcube/datatransfer/resolver/catalogue/resource/GetAllInfrastructureVREs.java Normal file
View File

@ -0,0 +1,213 @@
package org.gcube.datatransfer.resolver.catalogue.resource;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
/**
* The Class GetAllInfrastructureVREs.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
* Nov 8, 2018
*/
/**
*
* @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it
* Nov 8, 2018
*/
public class GetAllInfrastructureVREs {
public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureVREs.class);
protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()";
/**
* Load map of vre name to scope.
*
* @param rootScope the root scope
* @return the map of binding between (VRE_NAME, FULL_SCOPE_OF_VRE_NAME)
* @throws Exception
*/
public static Map<String, String> loadMapOFVreNameToScope(String rootScope) throws Exception{
String originalScope = ScopeProvider.instance.get();
try{
ScopeProvider.instance.set(rootScope);
String secondaryType = "INFRASTRUCTURE";
List<String> listVOScopes = getListOfVOScopes(secondaryType);
Map<String, String> vreNameFullScope = new HashMap<String,String>();
logger.info("Searching for secondaryType="+secondaryType +" scope/s found is/are: " +listVOScopes);
secondaryType = "VRE";
//int noVOTypeCount = 0;
for (String voScope : listVOScopes) {
int count = voScope.length() - voScope.replace("/", "").length();
//IS A VO
if(count==2){
logger.info(voScope +" is a VO...");
ScopeProvider.instance.set(voScope);
List<String> listVREs = getListOfResourcesForSecondaryType(secondaryType);
logger.debug("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs);
for (String vreName : listVREs) {
String vreScope = String.format("%s/%s", voScope,vreName);
vreNameFullScope.put(vreName, vreScope);
}
}else{
//noVOTypeCount++;
logger.info(voScope +" is not a VO, skipping it");
}
}
/*System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount));
for (String vreName : vreNameFullScope.keySet()) {
System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName));
}*/
logger.info("Total VRE is: "+vreNameFullScope.size());
return vreNameFullScope;
}catch(Exception e ){
throw new Exception("Error on loading the map of VRE nameto scope: ", e);
}
finally{
if(originalScope!=null && !originalScope.isEmpty()){
ScopeProvider.instance.set(originalScope);
}else
ScopeProvider.instance.reset();
}
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
private static List<String> getListOfResourcesForSecondaryType(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listResourceName = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
if(resourceNames!=null && resourceNames.size()>0)
listResourceName.add(resourceNames.get(0));
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listResourceName;
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
private static List<String> getListOfVOScopes(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listOfVOScopes = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
// List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
//
// if(resourceNames!=null && resourceNames.size()>0)
// listResourceName.add(resourceNames.get(0));
List<String> scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()");
for (String scopeFound : scopes) {
listOfVOScopes.add(scopeFound);
}
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listOfVOScopes;
}
}

6
src/main/java/org/gcube/datatransfer/resolver/listeners/UriResolverStartupListener.java
View File

@ -15,9 +15,10 @@ import javax.servlet.ServletContextListener;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.annotation.WebListener; import javax.servlet.annotation.WebListener;
import org.gcube.datatransfer.resolver.caches.LoadingCatalogueApplicationProfilesCache; //import org.gcube.datatransfer.resolver.caches.LoadingCatalogueApplicationProfilesCache;
import org.gcube.datatransfer.resolver.caches.LoadingGeonetworkInstanceCache; import org.gcube.datatransfer.resolver.caches.LoadingGeonetworkInstanceCache;
import org.gcube.datatransfer.resolver.caches.LoadingGisViewerApplicationURLCache; import org.gcube.datatransfer.resolver.caches.LoadingGisViewerApplicationURLCache;
import org.gcube.datatransfer.resolver.caches.LoadingVREsScopeCache;
import org.gcube.datatransfer.resolver.gis.property.ApplicationProfilePropertyReader; import org.gcube.datatransfer.resolver.gis.property.ApplicationProfilePropertyReader;
import org.gcube.datatransfer.resolver.gis.property.PropertyFileNotFoundException; import org.gcube.datatransfer.resolver.gis.property.PropertyFileNotFoundException;
import org.gcube.smartgears.ContextProvider; import org.gcube.smartgears.ContextProvider;
@ -79,7 +80,8 @@ public class UriResolverStartupListener implements ServletContextListener {
//init the caches //init the caches
new LoadingGeonetworkInstanceCache(); new LoadingGeonetworkInstanceCache();
new LoadingGisViewerApplicationURLCache(); new LoadingGisViewerApplicationURLCache();
new LoadingCatalogueApplicationProfilesCache(); new LoadingVREsScopeCache();
//new LoadingCatalogueApplicationProfilesCache();
logger.info("Context initialized with: "); logger.info("Context initialized with: ");
logger.info("Scope: "+rootContextScope); logger.info("Scope: "+rootContextScope);

36
src/main/java/org/gcube/datatransfer/resolver/services/CatalogueResolver.java
View File

@ -1,6 +1,7 @@
package org.gcube.datatransfer.resolver.services; package org.gcube.datatransfer.resolver.services;
import java.net.URL; import java.net.URL;
import java.util.concurrent.ExecutionException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.Consumes; import javax.ws.rs.Consumes;
@ -17,14 +18,12 @@ import javax.ws.rs.core.Response.Status;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datatransfer.resolver.ResourceCatalogueCodes; import org.gcube.datatransfer.resolver.ResourceCatalogueCodes;
import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileNotFoundException; import org.gcube.datatransfer.resolver.caches.LoadingVREsScopeCache;
import org.gcube.datatransfer.resolver.caches.LoadingCatalogueApplicationProfilesCache;
import org.gcube.datatransfer.resolver.catalogue.CatalogueRequest; import org.gcube.datatransfer.resolver.catalogue.CatalogueRequest;
import org.gcube.datatransfer.resolver.catalogue.resource.ApplicationProfileReaderForCatalogueResolver;
import org.gcube.datatransfer.resolver.catalogue.resource.CkanCatalogueConfigurationsReader; import org.gcube.datatransfer.resolver.catalogue.resource.CkanCatalogueConfigurationsReader;
import org.gcube.datatransfer.resolver.catalogue.resource.GatewayCKANCatalogueReference; import org.gcube.datatransfer.resolver.catalogue.resource.GatewayCKANCatalogueReference;
import org.gcube.datatransfer.resolver.catalogue.resource.UpdateApplicationProfileCatalogueResolver; import org.gcube.datatransfer.resolver.catalogue.resource.GetAllInfrastructureVREs;
import org.gcube.datatransfer.resolver.listeners.UriResolverStartupListener; import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.smartgears.utils.InnerMethodName; import org.gcube.smartgears.utils.InnerMethodName;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -35,6 +34,7 @@ import eu.trentorise.opendata.jackan.model.CkanDataset;
public class CatalogueResolver { public class CatalogueResolver {
private static Logger logger = LoggerFactory.getLogger(CatalogueResolver.class); private static Logger logger = LoggerFactory.getLogger(CatalogueResolver.class);
private static String helpURI = "https://wiki.gcube-system.org/gcube/URI_Resolver#CATALOGUE_Resolver";
/** The scope to enc decr. */ /** The scope to enc decr. */
//private String scopeToEncDecr = null; //private String scopeToEncDecr = null;
@ -51,14 +51,16 @@ public class CatalogueResolver {
//ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(vreName, true); //ApplicationProfileReaderForCatalogueResolver appPrCatResolver = new ApplicationProfileReaderForCatalogueResolver(vreName, true);
String fullScope = LoadingCatalogueApplicationProfilesCache.getCache().get(vreName); String fullScope = LoadingVREsScopeCache.getCache().get(vreName);
logger.debug("Read fullScope: "+fullScope + " for VRE_NAME: "+vreName +" into Application Profile "+ApplicationProfileReaderForCatalogueResolver.RESOURCE_NAME); logger.info("Read fullScope: "+fullScope + " for VRE_NAME: "+vreName +" from cache created by: "+GetAllInfrastructureVREs.class.getSimpleName());
//String fullScope = appPrCatResolver.getHashVreNameScope().get(vreName); //String fullScope = appPrCatResolver.getHashVreNameScope().get(vreName);
ScopeProvider.instance.set(fullScope); ScopeProvider.instance.set(fullScope);
GatewayCKANCatalogueReference ckanCatalogueReference = CkanCatalogueConfigurationsReader.loadCatalogueEndPoints(); GatewayCKANCatalogueReference ckanCatalogueReference = CkanCatalogueConfigurationsReader.loadCatalogueEndPoints();
logger.info("For scope "+fullScope+" loaded end points: "+ckanCatalogueReference);
//IS THE PRODUCT PLUBLIC OR PRIVATE? //IS THE PRODUCT PLUBLIC OR PRIVATE?
//USING ACCESS TO PUBLIC PORTLET IF THE ITEM IS PUBLIC, OTHERWISE ACCESS TO PRIVATE PORTLET //USING ACCESS TO PUBLIC PORTLET IF THE ITEM IS PUBLIC, OTHERWISE ACCESS TO PRIVATE PORTLET
String ckanPorltetUrl = ckanCatalogueReference.getPrivatePortletURL(); String ckanPorltetUrl = ckanCatalogueReference.getPrivatePortletURL();
@ -107,20 +109,32 @@ public class CatalogueResolver {
String serverUrl = getServerURL(req); String serverUrl = getServerURL(req);
final String vreName = scope.substring(scope.lastIndexOf("/")+1, scope.length()); final String vreName = scope.substring(scope.lastIndexOf("/")+1, scope.length());
String fullScope = null;
//CHECK IF THE vreName has a valid scope, so it is a valid VRE
try {
fullScope = LoadingVREsScopeCache.getCache().get(vreName);
}
catch (ExecutionException e1) {
logger.error("Error on getting full scope for vre name: "+vreName, e1);
}
if(fullScope==null)
ExceptionManager.throwWrongParameterException(req, "The scope '"+scope+"' does not matching any scope in the infrastructure. Is it valid?", this.getClass(), helpURI);
ResourceCatalogueCodes rc = ResourceCatalogueCodes.valueOfCodeValue(jsonRequest.getEntity_context()); ResourceCatalogueCodes rc = ResourceCatalogueCodes.valueOfCodeValue(jsonRequest.getEntity_context());
if(rc==null){ if(rc==null){
logger.error("Entity context is null/malformed"); logger.error("Entity context is null/malformed");
throw new WebApplicationException("Entity context is null/malformed", Status.BAD_REQUEST); ExceptionManager.throwBadRequestException(req, "Entity context is null/malformed", this.getClass(), helpURI);
//throw new WebApplicationException("Entity context is null/malformed", Status.BAD_REQUEST);
} }
String linkURL = String.format("%s/%s/%s/%s", serverUrl, rc.getId(), vreName, jsonRequest.getEntity_name()); String linkURL = String.format("%s/%s/%s/%s", serverUrl, rc.getId(), vreName, jsonRequest.getEntity_name());
logger.info("Writing Decoded Catalogue Link: "+linkURL); logger.info("Returining Catalogue URL: "+linkURL);
//IT'S GOING TO UPDATE THE GENERIC RESOURCE IF IS NEEDED //IT'S GOING TO UPDATE THE GENERIC RESOURCE IF IS NEEDED
final String fullscope = scope; /*final String fullscope = scope;
new Thread(){ new Thread(){
public void run() { public void run() {
try { try {
@ -133,7 +147,7 @@ public class CatalogueResolver {
logger.error("Error during validating Application Profile", e); logger.error("Error during validating Application Profile", e);
} }
}; };
}.start(); }.start();*/
return Response.ok(linkURL).build(); return Response.ok(linkURL).build();

204
src/test/java/GetAllInfrastructureVREs.java Normal file
View File

@ -0,0 +1,204 @@
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
/**
* The Class GetAllInfrastructureVREs.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
* Nov 8, 2018
*/
/**
*
* @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it
* Nov 8, 2018
*/
public class GetAllInfrastructureVREs {
public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureVREs.class);
protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()";
/**
* The main method.
*
* @param args the arguments
*/
public static void main(String[] args) {
//TODO TOKEN TO ROOT SCOPE
String rootScope = "/gcube";
//String rootScope = "/d4science.research-infrastructures.eu";
ScopeProvider.instance.set(rootScope);
String secondaryType = "INFRASTRUCTURE";
List<String> listVOScopes = getListOfVOScopes("INFRASTRUCTURE");
System.out.println("Searching for secondaryType="+secondaryType +" scope/s found is/are: " +listVOScopes);
Map<String, String> vreNameFullScope = new HashMap<String,String>();
secondaryType = "VRE";
int noVOTypeCount = 0;
for (String voScope : listVOScopes) {
int count = voScope.length() - voScope.replace("/", "").length();
//IS A VO
if(count==2){
logger.info(voScope +" is a VO...");
ScopeProvider.instance.set(voScope);
List<String> listVREs = getListOfResourcesForSecondaryType(secondaryType);
System.out.println("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs);
for (String vreName : listVREs) {
String vreScope = String.format("%s/%s", voScope,vreName);
vreNameFullScope.put(vreName, vreScope);
}
}else{
noVOTypeCount++;
System.out.println(voScope +" is not a VO, skipping it");
}
}
System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount));
for (String vreName : vreNameFullScope.keySet()) {
System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName));
}
System.out.println("Total VRE is: "+vreNameFullScope.size());
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
protected static List<String> getListOfResourcesForSecondaryType(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listResourceName = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
if(resourceNames!=null && resourceNames.size()>0)
listResourceName.add(resourceNames.get(0));
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listResourceName;
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
protected static List<String> getListOfVOScopes(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listOfVOScopes = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
// List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
//
// if(resourceNames!=null && resourceNames.size()>0)
// listResourceName.add(resourceNames.get(0));
List<String> scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()");
for (String scopeFound : scopes) {
listOfVOScopes.add(scopeFound);
}
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listOfVOScopes;
}
}

21
src/test/java/rest/TestResolvers.java
View File

@ -254,8 +254,8 @@ public class TestResolvers {
* *
* @throws Exception the exception * @throws Exception the exception
*/ */
//@Test @Test
public void testCatalogueResolverCreatePubrivatetemURL() throws Exception{ public void testCatalogueResolverCreatePubrivatItemURL() throws Exception{
String entityName = "dynamic_reporting"; String entityName = "dynamic_reporting";
//String entityContext = "product"; //String entityContext = "product";
@ -302,7 +302,7 @@ public class TestResolvers {
* *
* @throws Exception the exception * @throws Exception the exception
*/ */
@Test //@Test
public void testGisResolver() throws Exception{ public void testGisResolver() throws Exception{
String gisUUID = "55c19a1f-214b-4f81-9220-fba09fcfa91f"; String gisUUID = "55c19a1f-214b-4f81-9220-fba09fcfa91f";
@ -341,21 +341,6 @@ public class TestResolvers {
} }
/** /**
* Gets the content reponse. * Gets the content reponse.
* *