From 4113507376eef892f206b675b6637565e75ec8e7 Mon Sep 17 00:00:00 2001 From: "francesco.mangiacrapa" Date: Mon, 13 Jun 2022 15:04:04 +0200 Subject: [PATCH] Removed -SNAPSHOT to be released --- CHANGELOG.md | 2 +- pom.xml | 2 +- ...CatalogueMatching_SCOPE_CKANCONNECTOR.java | 158 +++++++++++++ .../datatransfer/test/StorageHubTest.java | 15 +- .../java/resources/GcoreEndpointReader.java | 161 +++++++++++++ .../resources/GetAllInfrastructureScopes.java | 221 ++++++++++++++++++ 6 files changed, 543 insertions(+), 16 deletions(-) create mode 100644 src/test/java/catalogue/CatalogueMatching_SCOPE_CKANCONNECTOR.java create mode 100644 src/test/java/resources/GcoreEndpointReader.java create mode 100644 src/test/java/resources/GetAllInfrastructureScopes.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 428bc70..79b0ac6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [v2.8.1-SNAPSHOT] - 2022-06-13 +## [v2.8.1] - 2022-06-13 **New** diff --git a/pom.xml b/pom.xml index 3ff57c4..24dc8d2 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.gcube.data.transfer uri-resolver - 2.8.1-SNAPSHOT + 2.8.1 war The URI Resolver is an HTTP URI resolver implemented as a REST service which gives access trough HTTP to different gcube Resolvers and gCube Applications. diff --git a/src/test/java/catalogue/CatalogueMatching_SCOPE_CKANCONNECTOR.java b/src/test/java/catalogue/CatalogueMatching_SCOPE_CKANCONNECTOR.java new file mode 100644 index 0000000..822d051 --- /dev/null +++ b/src/test/java/catalogue/CatalogueMatching_SCOPE_CKANCONNECTOR.java @@ -0,0 +1,158 @@ +package catalogue; + +/** + * + */ + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.spatial.data.geonetwork.LoginLevel; +import org.junit.Test; + +import resources.GcoreEndpointReader; +import resources.GetAllInfrastructureScopes; + +/** + * The Class CatalogueMatching_SCOPE_CKANCONNECTOR. + * + * @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it + * + * Jun 7, 2022 + */ +public class CatalogueMatching_SCOPE_CKANCONNECTOR { + + // GN CONFIGURATIONS + static String rootScope = "/d4science.research-infrastructures.eu"; + // static String rootScope = "/pred4s"; +// static String rootScope = "/gcube"; + static LoginLevel loginLevel = LoginLevel.CKAN; + + static PrintWriter reportPrintWriter; + static PrintWriter errorPrintWriter; + + private static int c = 0; + + /** + * Test catalogue discovery. + * + * @throws Exception the exception + */ + @Test + public void testCatalogueDiscovery() throws Exception { + + try { + FileWriter reportWriter = new FileWriter(rootScope.substring(1, rootScope.length()) + "_report_" + + CatalogueMatching_SCOPE_CKANCONNECTOR.class.getSimpleName() + ".csv", true); + FileWriter errorWriter = new FileWriter(rootScope.substring(1, rootScope.length()) + "_error_" + + CatalogueMatching_SCOPE_CKANCONNECTOR.class.getSimpleName() + ".csv", true); + BufferedWriter reportBW = new BufferedWriter(reportWriter); + BufferedWriter errorBW = new BufferedWriter(errorWriter); + reportPrintWriter = new PrintWriter(reportBW); + reportPrintWriter.println("NB.; SCOPE; CKAN_CONNECTOR_URL;"); + errorPrintWriter = new PrintWriter(errorBW); + errorPrintWriter.println("NB.; SCOPE; ERROR MESSAGE;"); + } catch (Exception e) { + e.printStackTrace(); + } + +// final Path destination = Paths.get("report_matching_gn_catalogue.csv"); +// Files.w(re, destination); + + List scopesProd = new ArrayList(); + + // RuntimeResourceReader readerRR; + try { + // MODE-1 +// readerRR = new RuntimeResourceReader(rootScope, platformName, null); +// System.out.println(readerRR.toString()); +// System.out.println("Using GN: "+readerRR.getParameters()); +// System.out.println("Scopes are: "+readerRR.getScopes().size()); +// scopesProd.addAll(readerRR.getScopes()); + + // MODE-2 + Map mapScopes = GetAllInfrastructureScopes.loadMapOfScopeNameToFullScope(rootScope); + scopesProd.addAll(mapScopes.values()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + System.out.println("Read " + scopesProd.size() + " VREs from root scope: " + rootScope); + for (String vre : scopesProd) { + System.out.println("VRE: " + vre); + } + + int count = 0; + try { + + final String ckanResource = "org.gcube.data.access.ckanconnector.CkanConnector"; + final String serviceName = "CkanConnector"; + final String serviceClass = "DataAccess"; + + int totalScope = scopesProd.size(); + int reportOp = 0; + int errorOp = 0; + for (String scope : scopesProd) { + count++; + System.out.println("#### Operation "+ count+ " of "+totalScope+") Fetching scope: " + scope); + ScopeProvider.instance.set(scope); + GcoreEndpointReader reader = null; + try { + // discovery = new GCatClientDiscovery(); + reader = new GcoreEndpointReader(scope, serviceClass, serviceName, ckanResource); + } catch (Exception e) { + System.out.println("gCat not instanciable in the scope: " + scope); + errorOp++; + writeError(errorOp + "; " + scope + " ; " + e.getMessage()); + } + + if (reader != null) { + reportOp++; + writeReport(reportOp + "; " + scope + " ; " + reader.getEndPoints().get(0) + " ; "); + } + + System.out.println("Sleeping..."); + Thread.sleep(500); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (reportPrintWriter != null) + reportPrintWriter.close(); + + if (errorPrintWriter != null) + errorPrintWriter.close(); + + System.out.println("Performed fetching from " + count + " scopes"); + System.out.println("FINISHED!!!"); + } + + } + + /** + * Write report. + * + * @param newline the newline + */ + private static synchronized void writeReport(String newline) { + c++; + reportPrintWriter.println(newline); + } + + /** + * Write error. + * + * @param newline the newline + */ + private static synchronized void writeError(String newline) { + c++; + errorPrintWriter.println(newline); + } + +} diff --git a/src/test/java/org/gcube/datatransfer/test/StorageHubTest.java b/src/test/java/org/gcube/datatransfer/test/StorageHubTest.java index 7e146c2..485047a 100644 --- a/src/test/java/org/gcube/datatransfer/test/StorageHubTest.java +++ b/src/test/java/org/gcube/datatransfer/test/StorageHubTest.java @@ -1,23 +1,10 @@ package org.gcube.datatransfer.test; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.ResponseBuilder; - import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.storagehub.client.StreamDescriptor; import org.gcube.common.storagehub.client.plugins.AbstractPlugin; import org.gcube.common.storagehub.client.proxies.ItemManagerClient; -import org.gcube.datatransfer.resolver.catalogue.resource.CkanCatalogueConfigurationsReader; -import org.gcube.datatransfer.resolver.catalogue.resource.GatewayCKANCatalogueReference; import org.gcube.datatransfer.resolver.init.UriResolverSmartGearManagerInit; -import org.gcube.datatransfer.resolver.services.CatalogueResolver; -import org.gcube.datatransfer.resolver.services.StorageHubResolver; -import org.gcube.datatransfer.resolver.services.error.ExceptionManager; -import org.gcube.datatransfer.resolver.shub.StorageHubMetadataResponseBuilder; -import org.junit.Before; -import org.junit.Test; - -import com.itextpdf.text.log.SysoCounter; /** @@ -34,7 +21,7 @@ public class StorageHubTest { private static String vreName = "BlueBridgeProject"; private String rootContextScope = "/d4science.research-infrastructures.eu"; - private String authorizationToken = "ea16e0fa-722a-4589-83b0-0a731d2d4039-843339462"; + private String authorizationToken = ""; //@Before public void init() { diff --git a/src/test/java/resources/GcoreEndpointReader.java b/src/test/java/resources/GcoreEndpointReader.java new file mode 100644 index 0000000..ee851e3 --- /dev/null +++ b/src/test/java/resources/GcoreEndpointReader.java @@ -0,0 +1,161 @@ +package resources; +import static org.gcube.resources.discovery.icclient.ICFactory.client; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * The Class GcoreEndpointReader. + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Jun 10, 2016 + */ +public class GcoreEndpointReader implements Serializable{ + + private static final long serialVersionUID = 7631710353375893823L; + + private static final Logger logger = LoggerFactory.getLogger(GcoreEndpointReader.class); + private List endPoints; + + /** + * Instantiates a new gcore endpoint reader. + * + * @param scope the scope + * @throws Exception the exception + */ + public GcoreEndpointReader(String scope, String serviceClass, String serviceName, String ckanResource) throws Exception { + + String currentScope = ScopeProvider.instance.get(); + try{ + + logger.info("set scope "+scope); + ScopeProvider.instance.set(scope); + + SimpleQuery query = queryFor(GCoreEndpoint.class); + query.addCondition(String.format("$resource/Profile/ServiceClass/text() eq '%s'",serviceClass)); + query.addCondition("$resource/Profile/DeploymentData/Status/text() eq 'ready'"); + query.addCondition(String.format("$resource/Profile/ServiceName/text() eq '%s'",serviceName)); + query.addCondition(String.format("$resource/Scopes/Scope/text()[.='%s']", scope)); // i.e. check the resource contains among the scopes this one + query.setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces//Endpoint[@EntryName/string() eq \""+ckanResource+"\"]/text()"); + + logger.debug("submitting quey "+query.toString()); + + DiscoveryClient client = client(); + List endpoints = client.submit(query); + if (endpoints == null || endpoints.isEmpty()) throw new Exception("Cannot retrieve the GCoreEndpoint serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope); + + + for (String endpoint : endpoints) { + System.out.println("end point "+endpoint); + } + this.endPoints = endpoints; + if(endPoints==null) + throw new Exception("Endpoint:"+ckanResource+", is null for serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope); + + logger.info("found entyname "+endPoints+" for ckanResource: "+ckanResource); + + /*Group accessPoints = se.profile().endpoints(); + if(accessPoints.size()==0) throw new Exception("Endpoint in serviceName serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope +" not found"); + + Endpoint ep = accessPoints.iterator().next(); + + String epName = ep.name(); + + System.out.println(epName);*/ + + }catch(Exception e){ + String error = "An error occurred during GCoreEndpoint discovery, serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope +"."; + logger.error(error, e); + throw new Exception(error); + }finally{ + logger.info("scope provider reset"); + ScopeProvider.instance.set(currentScope); + } + } + + + /** + * Gets the end points. + * + * @return the end points + */ + public List getEndPoints() { + + return endPoints; + } + +// private static String[] scopes = {"" + +// "/d4science.research-infrastructures.eu/gCubeApps", +// "/d4science.research-infrastructures.eu/FARM", +// "/d4science.research-infrastructures.eu/D4Research", +// "/d4science.research-infrastructures.eu/OpenAIRE", +// "/d4science.research-infrastructures.eu/Edison", +// "/d4science.research-infrastructures.eu/SmartArea", +// "/d4science.research-infrastructures.eu/SoBigData"}; + static PrintWriter reportPrintWriter; + static PrintWriter errorPrintWriter; + + /** + * The main method. + * + * @param args the arguments + */ + public static void main(String[] args) { + FileWriter reportWriter = null; + + final String ckanResource = "org.gcube.data.access.ckanconnector.CkanConnector"; + final String serviceName = "CkanConnector"; + final String serviceClass = "DataAccess"; + + final String rootScope = "/d4science.research-infrastructures.eu"; + + try{ + reportWriter = new FileWriter("report_ckanconnector.csv", true); + BufferedWriter reportBW = new BufferedWriter(reportWriter); + reportPrintWriter = new PrintWriter(reportBW); + reportPrintWriter.println("NB.; SCOPE; END POINTS;"); + + try { + int i = 0; + Map scopes = GetAllInfrastructureScopes.loadMapOfScopeNameToFullScope(rootScope); + for (String scope : scopes.values()) { + System.out.println("Ckecking scope: "+scope); + ScopeProvider.instance.set(scope); + try{ + i++; + GcoreEndpointReader reader = new GcoreEndpointReader(scope, serviceClass, serviceName, ckanResource); + System.out.println("Appending endpoints: "+reader.getEndPoints()); + reportPrintWriter.println(i+"; "+scope+"; "+reader.getEndPoints()); + }catch(Exception e){ + i--; + e.printStackTrace(); + } + } + } + catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + }catch(Exception e){ + e.printStackTrace(); + } + finally{ + if(reportPrintWriter!=null) + reportPrintWriter.close(); + } + } +} diff --git a/src/test/java/resources/GetAllInfrastructureScopes.java b/src/test/java/resources/GetAllInfrastructureScopes.java new file mode 100644 index 0000000..5eb5921 --- /dev/null +++ b/src/test/java/resources/GetAllInfrastructureScopes.java @@ -0,0 +1,221 @@ +package resources; +import static org.gcube.resources.discovery.icclient.ICFactory.client; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.gcube.common.resources.gcore.utils.XPathHelper; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.Query; +import org.gcube.resources.discovery.client.queries.impl.QueryBox; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; + + +/** + * The Class GetAllInfrastructureScopes. + * + * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) + * + * May 16, 2019 + */ +public class GetAllInfrastructureScopes { + + public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureScopes.class); + + protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()"; + + + /** + * Load map of scope name to full scope. + * + * @param rootScope the root scope + * @return the map of binding between (VRE_NAME, FULL_SCOPE_OF_VRE_NAME) + * @throws Exception the exception + */ + public static Map loadMapOfScopeNameToFullScope(String rootScope) throws Exception{ + + String originalScope = ScopeProvider.instance.get(); + + try{ + ScopeBean scopeBean = null; + Map scopeNameToFullScopeMap = new HashMap(); + ScopeProvider.instance.set(rootScope); + String secondaryType = Type.INFRASTRUCTURE.name(); + scopeBean = new ScopeBean(rootScope); + logger.info("Added the couple ({},{}) as {} (NAME, FULL SCOPE) into map", scopeBean.name(), rootScope, secondaryType); + scopeNameToFullScopeMap.put(scopeBean.name(), rootScope); + List listVOScopes = getListOfVOScopes(secondaryType); + logger.info("Searching for secondaryType={} the scope/s found is/are: " +secondaryType, listVOScopes); + + //int noVOTypeCount = 0; + for (String voScope : listVOScopes) { + //int count = voScope.length() - voScope.replace("/", "").length(); + scopeBean = new ScopeBean(voScope); + //IS A VO + //if(count==2){ + if(scopeBean.is(Type.VO)){ + secondaryType = Type.VO.name(); + logger.info("{} is a {}...",voScope,secondaryType); + ScopeProvider.instance.set(voScope); + scopeBean = new ScopeBean(voScope); + logger.info("Added the couple ({},{}) as {} (NAME, FULL SCOPE) into map", scopeBean.name(), voScope, secondaryType); + scopeNameToFullScopeMap.put(scopeBean.name(), voScope); + secondaryType = Type.VRE.name(); + List listVREs = getListOfResourcesForSecondaryType(secondaryType); + logger.debug("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs); + for (String vreName : listVREs) { + String vreScope = String.format("%s/%s", voScope,vreName); + scopeNameToFullScopeMap.put(vreName, vreScope); + } + + }else{ + //noVOTypeCount++; + logger.info(voScope +" is not a VO, skipping it"); + } + } + + /*System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount)); + for (String vreName : vreNameFullScope.keySet()) { + System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName)); + }*/ + + logger.info("Total VRE is: "+scopeNameToFullScopeMap.size()); + return scopeNameToFullScopeMap; + + }catch(Exception e ){ + throw new Exception("Error on loading the map of VRE nameto scope: ", e); + } + finally{ + if(originalScope!=null && !originalScope.isEmpty()){ + ScopeProvider.instance.set(originalScope); + }else + ScopeProvider.instance.reset(); + } + } + + + /** + * Gets the list of resources for secondary type. + * + * @param secondaryType the secondary type + * @return the list of resource names for the input secondary type + */ + private static List getListOfResourcesForSecondaryType(String secondaryType) { + + String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile"; + + List listResourceName = new ArrayList(); + + try { + logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType); + Query q = new QueryBox(queryString); + DiscoveryClient client = client(); + List listGenericResources = client.submit(q); + + logger.info("# of GenericResource returned are: "+listGenericResources.size()); + + if (listGenericResources == null || listGenericResources.size() == 0) + throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get()); + else { + + + for (String genericResource : listGenericResources) { + try{ + String elem = genericResource; + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = docBuilder.parse(new InputSource(new StringReader(elem))); + Element rootElement = document.getDocumentElement(); + XPathHelper helper = new XPathHelper(rootElement); + List resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT); + + if(resourceNames!=null && resourceNames.size()>0) + listResourceName.add(resourceNames.get(0)); + + }catch(Exception e){ + throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get()); + } + } + + } + + } catch (Exception e) { + logger.error("Error while trying to fetch generic resource from the infrastructure", e); + } + + return listResourceName; + + } + + + /** + * Gets the list of resources for secondary type. + * + * @param secondaryType the secondary type + * @return the list of resource names for the input secondary type + */ + private static List getListOfVOScopes(String secondaryType) { + + String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " + + "where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile"; + + List listOfVOScopes = new ArrayList(); + + try { + logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType); + Query q = new QueryBox(queryString); + DiscoveryClient client = client(); + List listGenericResources = client.submit(q); + + logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size()); + + if (listGenericResources == null || listGenericResources.size() == 0) + throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get()); + else { + + + for (String genericResource : listGenericResources) { + try{ + String elem = genericResource; + DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document document = docBuilder.parse(new InputSource(new StringReader(elem))); + Element rootElement = document.getDocumentElement(); + XPathHelper helper = new XPathHelper(rootElement); +// List resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT); +// +// if(resourceNames!=null && resourceNames.size()>0) +// listResourceName.add(resourceNames.get(0)); + + List scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()"); + for (String scopeFound : scopes) { + listOfVOScopes.add(scopeFound); + } + + }catch(Exception e){ + throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get()); + } + } + + } + + } catch (Exception e) { + logger.error("Error while trying to fetch generic resource from the infrastructure", e); + } + + return listOfVOScopes; + + } +}