Removed -SNAPSHOT to be released

This commit is contained in:
Francesco Mangiacrapa 2022-06-13 15:04:04 +02:00
parent c1abd152e3
commit 4113507376
6 changed files with 543 additions and 16 deletions

View File

@ -4,7 +4,7 @@
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [v2.8.1-SNAPSHOT] - 2022-06-13
## [v2.8.1] - 2022-06-13
**New**

View File

@ -9,7 +9,7 @@
</parent>
<groupId>org.gcube.data.transfer</groupId>
<artifactId>uri-resolver</artifactId>
<version>2.8.1-SNAPSHOT</version>
<version>2.8.1</version>
<packaging>war</packaging>
<description>The URI Resolver is an HTTP URI resolver implemented as a REST service which gives access trough HTTP to different gcube Resolvers and gCube Applications.</description>

View File

@ -0,0 +1,158 @@
package catalogue;
/**
*
*/
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.spatial.data.geonetwork.LoginLevel;
import org.junit.Test;
import resources.GcoreEndpointReader;
import resources.GetAllInfrastructureScopes;
/**
* The Class CatalogueMatching_SCOPE_CKANCONNECTOR.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Jun 7, 2022
*/
public class CatalogueMatching_SCOPE_CKANCONNECTOR {
// GN CONFIGURATIONS
static String rootScope = "/d4science.research-infrastructures.eu";
// static String rootScope = "/pred4s";
// static String rootScope = "/gcube";
static LoginLevel loginLevel = LoginLevel.CKAN;
static PrintWriter reportPrintWriter;
static PrintWriter errorPrintWriter;
private static int c = 0;
/**
* Test catalogue discovery.
*
* @throws Exception the exception
*/
@Test
public void testCatalogueDiscovery() throws Exception {
try {
FileWriter reportWriter = new FileWriter(rootScope.substring(1, rootScope.length()) + "_report_"
+ CatalogueMatching_SCOPE_CKANCONNECTOR.class.getSimpleName() + ".csv", true);
FileWriter errorWriter = new FileWriter(rootScope.substring(1, rootScope.length()) + "_error_"
+ CatalogueMatching_SCOPE_CKANCONNECTOR.class.getSimpleName() + ".csv", true);
BufferedWriter reportBW = new BufferedWriter(reportWriter);
BufferedWriter errorBW = new BufferedWriter(errorWriter);
reportPrintWriter = new PrintWriter(reportBW);
reportPrintWriter.println("NB.; SCOPE; CKAN_CONNECTOR_URL;");
errorPrintWriter = new PrintWriter(errorBW);
errorPrintWriter.println("NB.; SCOPE; ERROR MESSAGE;");
} catch (Exception e) {
e.printStackTrace();
}
// final Path destination = Paths.get("report_matching_gn_catalogue.csv");
// Files.w(re, destination);
List<String> scopesProd = new ArrayList<String>();
// RuntimeResourceReader readerRR;
try {
// MODE-1
// readerRR = new RuntimeResourceReader(rootScope, platformName, null);
// System.out.println(readerRR.toString());
// System.out.println("Using GN: "+readerRR.getParameters());
// System.out.println("Scopes are: "+readerRR.getScopes().size());
// scopesProd.addAll(readerRR.getScopes());
// MODE-2
Map<String, String> mapScopes = GetAllInfrastructureScopes.loadMapOfScopeNameToFullScope(rootScope);
scopesProd.addAll(mapScopes.values());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Read " + scopesProd.size() + " VREs from root scope: " + rootScope);
for (String vre : scopesProd) {
System.out.println("VRE: " + vre);
}
int count = 0;
try {
final String ckanResource = "org.gcube.data.access.ckanconnector.CkanConnector";
final String serviceName = "CkanConnector";
final String serviceClass = "DataAccess";
int totalScope = scopesProd.size();
int reportOp = 0;
int errorOp = 0;
for (String scope : scopesProd) {
count++;
System.out.println("#### Operation "+ count+ " of "+totalScope+") Fetching scope: " + scope);
ScopeProvider.instance.set(scope);
GcoreEndpointReader reader = null;
try {
// discovery = new GCatClientDiscovery();
reader = new GcoreEndpointReader(scope, serviceClass, serviceName, ckanResource);
} catch (Exception e) {
System.out.println("gCat not instanciable in the scope: " + scope);
errorOp++;
writeError(errorOp + "; " + scope + " ; " + e.getMessage());
}
if (reader != null) {
reportOp++;
writeReport(reportOp + "; " + scope + " ; " + reader.getEndPoints().get(0) + " ; ");
}
System.out.println("Sleeping...");
Thread.sleep(500);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (reportPrintWriter != null)
reportPrintWriter.close();
if (errorPrintWriter != null)
errorPrintWriter.close();
System.out.println("Performed fetching from " + count + " scopes");
System.out.println("FINISHED!!!");
}
}
/**
* Write report.
*
* @param newline the newline
*/
private static synchronized void writeReport(String newline) {
c++;
reportPrintWriter.println(newline);
}
/**
* Write error.
*
* @param newline the newline
*/
private static synchronized void writeError(String newline) {
c++;
errorPrintWriter.println(newline);
}
}

View File

@ -1,23 +1,10 @@
package org.gcube.datatransfer.test;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.storagehub.client.StreamDescriptor;
import org.gcube.common.storagehub.client.plugins.AbstractPlugin;
import org.gcube.common.storagehub.client.proxies.ItemManagerClient;
import org.gcube.datatransfer.resolver.catalogue.resource.CkanCatalogueConfigurationsReader;
import org.gcube.datatransfer.resolver.catalogue.resource.GatewayCKANCatalogueReference;
import org.gcube.datatransfer.resolver.init.UriResolverSmartGearManagerInit;
import org.gcube.datatransfer.resolver.services.CatalogueResolver;
import org.gcube.datatransfer.resolver.services.StorageHubResolver;
import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.datatransfer.resolver.shub.StorageHubMetadataResponseBuilder;
import org.junit.Before;
import org.junit.Test;
import com.itextpdf.text.log.SysoCounter;
/**
@ -34,7 +21,7 @@ public class StorageHubTest {
private static String vreName = "BlueBridgeProject";
private String rootContextScope = "/d4science.research-infrastructures.eu";
private String authorizationToken = "ea16e0fa-722a-4589-83b0-0a731d2d4039-843339462";
private String authorizationToken = "";
//@Before
public void init() {

View File

@ -0,0 +1,161 @@
package resources;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class GcoreEndpointReader.
*
* @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it
* Jun 10, 2016
*/
public class GcoreEndpointReader implements Serializable{
private static final long serialVersionUID = 7631710353375893823L;
private static final Logger logger = LoggerFactory.getLogger(GcoreEndpointReader.class);
private List<String> endPoints;
/**
* Instantiates a new gcore endpoint reader.
*
* @param scope the scope
* @throws Exception the exception
*/
public GcoreEndpointReader(String scope, String serviceClass, String serviceName, String ckanResource) throws Exception {
String currentScope = ScopeProvider.instance.get();
try{
logger.info("set scope "+scope);
ScopeProvider.instance.set(scope);
SimpleQuery query = queryFor(GCoreEndpoint.class);
query.addCondition(String.format("$resource/Profile/ServiceClass/text() eq '%s'",serviceClass));
query.addCondition("$resource/Profile/DeploymentData/Status/text() eq 'ready'");
query.addCondition(String.format("$resource/Profile/ServiceName/text() eq '%s'",serviceName));
query.addCondition(String.format("$resource/Scopes/Scope/text()[.='%s']", scope)); // i.e. check the resource contains among the scopes this one
query.setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces//Endpoint[@EntryName/string() eq \""+ckanResource+"\"]/text()");
logger.debug("submitting quey "+query.toString());
DiscoveryClient<String> client = client();
List<String> endpoints = client.submit(query);
if (endpoints == null || endpoints.isEmpty()) throw new Exception("Cannot retrieve the GCoreEndpoint serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope);
for (String endpoint : endpoints) {
System.out.println("end point "+endpoint);
}
this.endPoints = endpoints;
if(endPoints==null)
throw new Exception("Endpoint:"+ckanResource+", is null for serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope);
logger.info("found entyname "+endPoints+" for ckanResource: "+ckanResource);
/*Group<Endpoint> accessPoints = se.profile().endpoints();
if(accessPoints.size()==0) throw new Exception("Endpoint in serviceName serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope +" not found");
Endpoint ep = accessPoints.iterator().next();
String epName = ep.name();
System.out.println(epName);*/
}catch(Exception e){
String error = "An error occurred during GCoreEndpoint discovery, serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope +".";
logger.error(error, e);
throw new Exception(error);
}finally{
logger.info("scope provider reset");
ScopeProvider.instance.set(currentScope);
}
}
/**
* Gets the end points.
*
* @return the end points
*/
public List<String> getEndPoints() {
return endPoints;
}
// private static String[] scopes = {"" +
// "/d4science.research-infrastructures.eu/gCubeApps",
// "/d4science.research-infrastructures.eu/FARM",
// "/d4science.research-infrastructures.eu/D4Research",
// "/d4science.research-infrastructures.eu/OpenAIRE",
// "/d4science.research-infrastructures.eu/Edison",
// "/d4science.research-infrastructures.eu/SmartArea",
// "/d4science.research-infrastructures.eu/SoBigData"};
static PrintWriter reportPrintWriter;
static PrintWriter errorPrintWriter;
/**
* The main method.
*
* @param args the arguments
*/
public static void main(String[] args) {
FileWriter reportWriter = null;
final String ckanResource = "org.gcube.data.access.ckanconnector.CkanConnector";
final String serviceName = "CkanConnector";
final String serviceClass = "DataAccess";
final String rootScope = "/d4science.research-infrastructures.eu";
try{
reportWriter = new FileWriter("report_ckanconnector.csv", true);
BufferedWriter reportBW = new BufferedWriter(reportWriter);
reportPrintWriter = new PrintWriter(reportBW);
reportPrintWriter.println("NB.; SCOPE; END POINTS;");
try {
int i = 0;
Map<String, String> scopes = GetAllInfrastructureScopes.loadMapOfScopeNameToFullScope(rootScope);
for (String scope : scopes.values()) {
System.out.println("Ckecking scope: "+scope);
ScopeProvider.instance.set(scope);
try{
i++;
GcoreEndpointReader reader = new GcoreEndpointReader(scope, serviceClass, serviceName, ckanResource);
System.out.println("Appending endpoints: "+reader.getEndPoints());
reportPrintWriter.println(i+"; "+scope+"; "+reader.getEndPoints());
}catch(Exception e){
i--;
e.printStackTrace();
}
}
}
catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}catch(Exception e){
e.printStackTrace();
}
finally{
if(reportPrintWriter!=null)
reportPrintWriter.close();
}
}
}

View File

@ -0,0 +1,221 @@
package resources;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
/**
* The Class GetAllInfrastructureScopes.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* May 16, 2019
*/
public class GetAllInfrastructureScopes {
public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureScopes.class);
protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()";
/**
* Load map of scope name to full scope.
*
* @param rootScope the root scope
* @return the map of binding between (VRE_NAME, FULL_SCOPE_OF_VRE_NAME)
* @throws Exception the exception
*/
public static Map<String, String> loadMapOfScopeNameToFullScope(String rootScope) throws Exception{
String originalScope = ScopeProvider.instance.get();
try{
ScopeBean scopeBean = null;
Map<String, String> scopeNameToFullScopeMap = new HashMap<String,String>();
ScopeProvider.instance.set(rootScope);
String secondaryType = Type.INFRASTRUCTURE.name();
scopeBean = new ScopeBean(rootScope);
logger.info("Added the couple ({},{}) as {} (NAME, FULL SCOPE) into map", scopeBean.name(), rootScope, secondaryType);
scopeNameToFullScopeMap.put(scopeBean.name(), rootScope);
List<String> listVOScopes = getListOfVOScopes(secondaryType);
logger.info("Searching for secondaryType={} the scope/s found is/are: " +secondaryType, listVOScopes);
//int noVOTypeCount = 0;
for (String voScope : listVOScopes) {
//int count = voScope.length() - voScope.replace("/", "").length();
scopeBean = new ScopeBean(voScope);
//IS A VO
//if(count==2){
if(scopeBean.is(Type.VO)){
secondaryType = Type.VO.name();
logger.info("{} is a {}...",voScope,secondaryType);
ScopeProvider.instance.set(voScope);
scopeBean = new ScopeBean(voScope);
logger.info("Added the couple ({},{}) as {} (NAME, FULL SCOPE) into map", scopeBean.name(), voScope, secondaryType);
scopeNameToFullScopeMap.put(scopeBean.name(), voScope);
secondaryType = Type.VRE.name();
List<String> listVREs = getListOfResourcesForSecondaryType(secondaryType);
logger.debug("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs);
for (String vreName : listVREs) {
String vreScope = String.format("%s/%s", voScope,vreName);
scopeNameToFullScopeMap.put(vreName, vreScope);
}
}else{
//noVOTypeCount++;
logger.info(voScope +" is not a VO, skipping it");
}
}
/*System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount));
for (String vreName : vreNameFullScope.keySet()) {
System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName));
}*/
logger.info("Total VRE is: "+scopeNameToFullScopeMap.size());
return scopeNameToFullScopeMap;
}catch(Exception e ){
throw new Exception("Error on loading the map of VRE nameto scope: ", e);
}
finally{
if(originalScope!=null && !originalScope.isEmpty()){
ScopeProvider.instance.set(originalScope);
}else
ScopeProvider.instance.reset();
}
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
private static List<String> getListOfResourcesForSecondaryType(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listResourceName = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
if(resourceNames!=null && resourceNames.size()>0)
listResourceName.add(resourceNames.get(0));
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listResourceName;
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
private static List<String> getListOfVOScopes(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listOfVOScopes = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
// List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
//
// if(resourceNames!=null && resourceNames.size()>0)
// listResourceName.add(resourceNames.get(0));
List<String> scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()");
for (String scopeFound : scopes) {
listOfVOScopes.add(scopeFound);
}
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listOfVOScopes;
}
}