Compare commits

...

6 Commits

15 changed files with 685 additions and 95 deletions

View File

@ -5,7 +5,17 @@
<wb-module deploy-name="uri-resolver-2.7.2-SNAPSHOT">
<wb-module deploy-name="uri-resolver-2.8.1-SNAPSHOT">
@ -19,6 +29,11 @@
<wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/> <wb-resource deploy-path="/" source-path="/src/main/webapp" tag="defaultRootSource"/>
@ -26,6 +41,11 @@
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/> <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
@ -33,6 +53,11 @@
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/> <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
@ -40,6 +65,11 @@
<property name="context-root" value="uri-resolver"/> <property name="context-root" value="uri-resolver"/>
@ -47,6 +77,11 @@
<property name="java-output-path" value="/uri-resolver/target/classes"/> <property name="java-output-path" value="/uri-resolver/target/classes"/>
@ -54,6 +89,11 @@
</wb-module> </wb-module>
@ -61,4 +101,9 @@
</project-modules> </project-modules>

View File

@ -4,7 +4,24 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [v2-7.2] - 2022-04-06 ## [v2.8.1-SNAPSHOT] - 2022-06-13
**New**
- [#23156] CatalogueResolver enhanced to manage (create/resolve) an input query string
**Bug Fixing**
- [#23495] Removed the check on the gcube token stored in thread local. The working context of VRE will be checked from ScopeProvider
## [v2.7.3-SNAPSHOT] - 2022-04-27
**Bug Fixing**
- [#23213] Fixed NoClassDefFoundError: org/geotoolkit/xml/XML
## [v2.7.2] - 2022-04-06
**New** **New**
@ -18,7 +35,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- [#20743] Integration with javamelody - [#20743] Integration with javamelody
## [v2-6-1] - 2022-01-17 ## [v2.6.1] - 2022-01-17
**New features** **New features**
@ -30,7 +47,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
- [#21560] Checking the bug fix done - [#21560] Checking the bug fix done
- Moved to gcube-smartgears-bom.2.1.0 - Moved to gcube-smartgears-bom.2.1.0
## [v2-5-0] - 2021-04-08 ## [v2.5.0] - 2021-04-08
**New features** **New features**
@ -41,7 +58,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
[#21093] StorageHubResolver HEAD request does not support Content-Length [#21093] StorageHubResolver HEAD request does not support Content-Length
## [v2-4-1] - 2021-01-13 ## [v2.4.1] - 2021-01-13
**Bug Fixes** **Bug Fixes**

View File

@ -10,11 +10,17 @@
pattern="org.gcube.datatransfer.resolver.services.UriResolverIndex" /> pattern="org.gcube.datatransfer.resolver.services.UriResolverIndex" />
<exclude <exclude
pattern="org.gcube.datatransfer.resolver.services.UriResolverInfo" /> pattern="org.gcube.datatransfer.resolver.services.UriResolverInfo" />
<exclude pattern="org.gcube.datatransfer.resolver.services.OatResolver" /> <exclude
pattern="org.gcube.datatransfer.resolver.services.OatResolver" />
<exclude <exclude
pattern="org.gcube.datatransfer.resolver.services.tobackward.BackCatalogueResolver" /> pattern="org.gcube.datatransfer.resolver.services.tobackward.BackCatalogueResolver" />
<exclude <exclude
pattern="org.gcube.datatransfer.resolver.services.tobackward.BackUriResolverCatalogueResolver" /> pattern="org.gcube.datatransfer.resolver.services.tobackward.BackUriResolverCatalogueResolver" />
<exclude pattern="org.geotoolkit.internal.jaxb.referencing.*" />
<exclude pattern="org.opengis.metadata.*" />
</api-classes> </api-classes>
<modules> <modules>

10
pom.xml
View File

@ -9,7 +9,7 @@
</parent> </parent>
<groupId>org.gcube.data.transfer</groupId> <groupId>org.gcube.data.transfer</groupId>
<artifactId>uri-resolver</artifactId> <artifactId>uri-resolver</artifactId>
<version>2.7.2</version> <version>2.8.1-SNAPSHOT</version>
<packaging>war</packaging> <packaging>war</packaging>
<description>The URI Resolver is an HTTP URI resolver implemented as a REST service which gives access trough HTTP to different gcube Resolvers and gCube Applications.</description> <description>The URI Resolver is an HTTP URI resolver implemented as a REST service which gives access trough HTTP to different gcube Resolvers and gCube Applications.</description>
@ -250,14 +250,6 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- Required for Enunciate plugin -->
<dependency>
<groupId>org.geotoolkit</groupId>
<artifactId>geotk-xml-base</artifactId>
<version>3.20-geoapi-3.0</version>
<scope>provided</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>

View File

@ -5,15 +5,14 @@ package org.gcube.datatransfer.resolver.catalogue;
* *
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it * @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
* *
* Mar 24, 2022 * Apr 20, 2022
*
* Binding Catalogue Request as a JSON
*/ */
public class CatalogueRequest { public class CatalogueRequest {
private String gcube_scope; private String gcube_scope;
private String entity_context; private String entity_context;
private String entity_name; private String entity_name;
private String query_string;
/** /**
* Gets the gcube_scope. * Gets the gcube_scope.
@ -42,19 +41,22 @@ public class CatalogueRequest {
return entity_name; return entity_name;
} }
/**
* Gets the query string.
*
* @return the query string
*/
public String getQuery_string() {
return query_string;
}
/** /**
* To string. * To string.
* *
* @return the string * @return the string
*/ */
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override @Override
public String toString() { public String toString() {
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
builder.append("CatalogueRequest [gcube_scope="); builder.append("CatalogueRequest [gcube_scope=");
builder.append(gcube_scope); builder.append(gcube_scope);
@ -62,6 +64,8 @@ public class CatalogueRequest {
builder.append(entity_context); builder.append(entity_context);
builder.append(", entity_name="); builder.append(", entity_name=");
builder.append(entity_name); builder.append(entity_name);
builder.append(", query_string=");
builder.append(query_string);
builder.append("]"); builder.append("]");
return builder.toString(); return builder.toString();
} }

View File

@ -94,12 +94,12 @@ public class CatalogueStaticConfigurations{
if(toPrivateCKANCatalogueURL.startsWith(prefix)) { if(toPrivateCKANCatalogueURL.startsWith(prefix)) {
toPrivateCKANCatalogueURL = toPrivateCKANCatalogueURL.replaceFirst(prefix, ""); //removing prefix toPrivateCKANCatalogueURL = toPrivateCKANCatalogueURL.replaceFirst(prefix, ""); //removing prefix
String catalogueName = toPrivateCKANCatalogueURL.substring(0,toPrivateCKANCatalogueURL.indexOf(".d4science")); String catalogueName = toPrivateCKANCatalogueURL.substring(0,toPrivateCKANCatalogueURL.indexOf(".d4science"));
LOG.info("Catalogue Name extration returning value: "+catalogueName); LOG.info("Catalogue Name extraction returning value: "+catalogueName);
return catalogueName; return catalogueName;
} }
} }
LOG.info("Catalogue Name extration returning null"); LOG.info("Catalogue Name extraction returning null");
return null; return null;
} }

View File

@ -43,7 +43,6 @@ import org.gcube.common.storagehub.model.Metadata;
import org.gcube.data.analysis.dminvocation.ActionType; import org.gcube.data.analysis.dminvocation.ActionType;
import org.gcube.data.analysis.dminvocation.DataMinerInvocationManager; import org.gcube.data.analysis.dminvocation.DataMinerInvocationManager;
import org.gcube.data.analysis.dminvocation.model.DataMinerInvocation; import org.gcube.data.analysis.dminvocation.model.DataMinerInvocation;
import org.gcube.datatransfer.resolver.requesthandler.RequestHandler;
import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.datatransfer.resolver.services.exceptions.InternalServerException; import org.gcube.datatransfer.resolver.services.exceptions.InternalServerException;
import org.gcube.datatransfer.resolver.util.ScopeUtil; import org.gcube.datatransfer.resolver.util.ScopeUtil;
@ -119,20 +118,11 @@ public class AnalyticsCreateResolver {
logger.debug("The body contains the request: " + jsonRequest.toString()); logger.debug("The body contains the request: " + jsonRequest.toString());
String contextToken = SecurityTokenProvider.instance.get();
String scope = ScopeProvider.instance.get(); String scope = ScopeProvider.instance.get();
// logger.info("SecurityTokenProvider contextToken: "+contextToken); // logger.info("SecurityTokenProvider contextToken: "+contextToken);
logger.info("ScopeProvider has scope: " + scope); logger.info("ScopeProvider has scope: " + scope);
String appToken = req.getServletContext().getInitParameter(RequestHandler.ROOT_APP_TOKEN);
if (contextToken.compareTo(appToken) == 0) {
logger.error("Token not passed, SecurityTokenProvider contains the root app token: "
+ appToken.substring(0, 10) + "...");
throw ExceptionManager.unauthorizedException(req,
"You are not authorized. You must pass a token of VRE", this.getClass(), helpURI);
}
String operatorID = jsonRequest.getOperatorId(); String operatorID = jsonRequest.getOperatorId();
if (scope == null || scope.isEmpty()) { if (scope == null || scope.isEmpty()) {
@ -150,6 +140,13 @@ public class AnalyticsCreateResolver {
} }
ScopeBean scopeBean = new ScopeBean(scope); ScopeBean scopeBean = new ScopeBean(scope);
if (scopeBean.is(Type.INFRASTRUCTURE) || scopeBean.is(Type.VO)) {
logger.error(Type.INFRASTRUCTURE + "or " + Type.VO + " scope detected, you must set a scope of VRE");
throw ExceptionManager.unauthorizedException(req,
"You are not authorized. You must pass a token of VRE", this.getClass(), helpURI);
}
String publicLinkToDMInvFile = ""; String publicLinkToDMInvFile = "";
if (scopeBean.is(Type.VRE)) { if (scopeBean.is(Type.VRE)) {
@ -163,6 +160,7 @@ public class AnalyticsCreateResolver {
File tempInvocationFile = null; File tempInvocationFile = null;
try { try {
String contextToken = SecurityTokenProvider.instance.get();
String xmlRequest = DataMinerInvocationManager.getInstance().marshalingXML(jsonRequest, true, true); String xmlRequest = DataMinerInvocationManager.getInstance().marshalingXML(jsonRequest, true, true);
String uniqueName = createDMInvocationFileName(jsonRequest.getOperatorId()); String uniqueName = createDMInvocationFileName(jsonRequest.getOperatorId());
tempInvocationFile = createTempFile(uniqueName, ".xml", xmlRequest.getBytes()); tempInvocationFile = createTempFile(uniqueName, ".xml", xmlRequest.getBytes());
@ -186,8 +184,9 @@ public class AnalyticsCreateResolver {
StorageHubManagement storageHubManagement = new StorageHubManagement(); StorageHubManagement storageHubManagement = new StorageHubManagement();
Metadata metadata = new Metadata(); Metadata metadata = new Metadata();
Map<String, String> theMap = new HashMap<String, String>(); Map<String, Object> theMap = new HashMap<String, Object>();
theMap.put("owner", owner); theMap.put("owner", owner);
metadata.setMap(theMap);
URL thePublicLink = null; URL thePublicLink = null;
try { try {

View File

@ -18,6 +18,7 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import org.apache.commons.codec.binary.Base64;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.common.scope.impl.ScopeBean.Type;
@ -92,6 +93,7 @@ public class CatalogueResolver {
try { try {
InnerMethodName.instance.set("resolveCataloguePublicLink"); InnerMethodName.instance.set("resolveCataloguePublicLink");
ItemCatalogueURLs itemCatalogueURLs = getItemCatalogueURLs(req, vreName, entityContext, entityName); ItemCatalogueURLs itemCatalogueURLs = getItemCatalogueURLs(req, vreName, entityContext, entityName);
String itemCatalogueURL; String itemCatalogueURL;
@ -111,8 +113,19 @@ public class CatalogueResolver {
} }
} else { } else {
itemCatalogueURL = itemCatalogueURLs.getPrivateVRECataloguePortletURL(); itemCatalogueURL = itemCatalogueURLs.getPrivateVRECataloguePortletURL();
logger.info("The dataset " + itemCatalogueURLs.getItemName() logger.info("Either the " + itemCatalogueURLs.getItemName()
+ " is a private item (to VRE) so using protected access to CKAN portlet: " + itemCatalogueURL); + " is not an item (alias dataset) or it is a private item (to VRE) so using protected access to CKAN portlet: "
+ itemCatalogueURL);
}
String queryString = req.getQueryString();
if (queryString != null) {
logger.info("Query string found: " + queryString);
byte[] stringEncBytes = Base64.encodeBase64(queryString.getBytes());
String queryStringEnc = "query_string=" + new String(stringEncBytes);
logger.info("Adding Base64 encoded: " + queryString);
itemCatalogueURL += itemCatalogueURL.contains("?") ? "&" : "?";
itemCatalogueURL += queryStringEnc;
} }
return Response.seeOther(new URL(itemCatalogueURL).toURI()).build(); return Response.seeOther(new URL(itemCatalogueURL).toURI()).build();
@ -190,6 +203,11 @@ public class CatalogueResolver {
} }
String linkURL = String.format("%s/%s/%s/%s", serverUrl, rc.getId(), vreName, jsonRequest.getEntity_name()); String linkURL = String.format("%s/%s/%s/%s", serverUrl, rc.getId(), vreName, jsonRequest.getEntity_name());
if (jsonRequest.getQuery_string() != null) {
linkURL += "?" + jsonRequest.getQuery_string();
}
logger.info("Returining Catalogue URL: " + linkURL); logger.info("Returining Catalogue URL: " + linkURL);
return Response.ok(linkURL).header("Location", linkURL).build(); return Response.ok(linkURL).header("Location", linkURL).build();

View File

@ -13,11 +13,9 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.datatransfer.resolver.requesthandler.RequestHandler;
import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.datatransfer.resolver.util.Util; import org.gcube.datatransfer.resolver.util.Util;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -60,22 +58,18 @@ public class KnimeCreateResolver {
logger.info(this.getClass().getSimpleName() + " POST starts..."); logger.info(this.getClass().getSimpleName() + " POST starts...");
try { try {
String contextToken = SecurityTokenProvider.instance.get();
String scope = ScopeProvider.instance.get(); String scope = ScopeProvider.instance.get();
// logger.info("SecurityTokenProvider contextToken: "+contextToken); // logger.info("SecurityTokenProvider contextToken: "+contextToken);
logger.info("ScopeProvider has scope: " + scope); logger.info("ScopeProvider has scope: " + scope);
String appToken = req.getServletContext().getInitParameter(RequestHandler.ROOT_APP_TOKEN); ScopeBean scopeBean = new ScopeBean(scope);
if (scopeBean.is(Type.INFRASTRUCTURE) || scopeBean.is(Type.VO)) {
if (contextToken.compareTo(appToken) == 0) { logger.error(Type.INFRASTRUCTURE + "or " + Type.VO + " scope detected, you must set a scope of VRE");
logger.error("Token not passed, SecurityTokenProvider contains the root app token: "
+ appToken.substring(0, 10) + "...");
throw ExceptionManager.unauthorizedException(req, throw ExceptionManager.unauthorizedException(req,
"You are not authorized. You must pass a token of VRE", this.getClass(), helpURI); "You are not authorized. You must pass a token of VRE", this.getClass(), helpURI);
} }
ScopeBean scopeBean = new ScopeBean(scope);
if (scopeBean.is(Type.VRE)) { if (scopeBean.is(Type.VRE)) {
String vreName = scopeBean.name(); String vreName = scopeBean.name();

View File

@ -22,12 +22,12 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.encryption.StringEncrypter; import org.gcube.common.encryption.StringEncrypter;
import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.datatransfer.resolver.requesthandler.RequestHandler; import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.datatransfer.resolver.services.exceptions.NotFoundException; import org.gcube.datatransfer.resolver.services.exceptions.NotFoundException;
import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.api.DiscoveryClient;
@ -72,15 +72,12 @@ public class WekeoResolver {
try { try {
String contextToken = SecurityTokenProvider.instance.get();
String scope = ScopeProvider.instance.get(); String scope = ScopeProvider.instance.get();
logger.info("ScopeProvider has scope: " + scope); logger.info("ScopeProvider has scope: " + scope);
String appToken = req.getServletContext().getInitParameter(RequestHandler.ROOT_APP_TOKEN); ScopeBean scopeBean = new ScopeBean(scope);
if (scopeBean.is(Type.INFRASTRUCTURE) || scopeBean.is(Type.VO)) {
if (contextToken.compareTo(appToken) == 0) { logger.error(Type.INFRASTRUCTURE + "or " + Type.VO + " scope detected, you must set a scope of VRE");
logger.error("Token not passed, SecurityTokenProvider contains the root app token: "
+ appToken.substring(0, 10) + "...");
throw ExceptionManager.unauthorizedException(req, throw ExceptionManager.unauthorizedException(req,
"You are not authorized. You must pass a token of VRE", this.getClass(), helpURI); "You are not authorized. You must pass a token of VRE", this.getClass(), helpURI);
} }

View File

@ -0,0 +1,158 @@
package catalogue;
/**
*
*/
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.spatial.data.geonetwork.LoginLevel;
import org.junit.Test;
import resources.GcoreEndpointReader;
import resources.GetAllInfrastructureScopes;
/**
* The Class CatalogueMatching_SCOPE_CKANCONNECTOR.
*
* @author Francesco Mangiacrapa at ISTI-CNR francesco.mangiacrapa@isti.cnr.it
*
* Jun 7, 2022
*/
public class CatalogueMatching_SCOPE_CKANCONNECTOR {
// GN CONFIGURATIONS
static String rootScope = "/d4science.research-infrastructures.eu";
// static String rootScope = "/pred4s";
// static String rootScope = "/gcube";
static LoginLevel loginLevel = LoginLevel.CKAN;
static PrintWriter reportPrintWriter;
static PrintWriter errorPrintWriter;
private static int c = 0;
/**
* Test catalogue discovery.
*
* @throws Exception the exception
*/
@Test
public void testCatalogueDiscovery() throws Exception {
try {
FileWriter reportWriter = new FileWriter(rootScope.substring(1, rootScope.length()) + "_report_"
+ CatalogueMatching_SCOPE_CKANCONNECTOR.class.getSimpleName() + ".csv", true);
FileWriter errorWriter = new FileWriter(rootScope.substring(1, rootScope.length()) + "_error_"
+ CatalogueMatching_SCOPE_CKANCONNECTOR.class.getSimpleName() + ".csv", true);
BufferedWriter reportBW = new BufferedWriter(reportWriter);
BufferedWriter errorBW = new BufferedWriter(errorWriter);
reportPrintWriter = new PrintWriter(reportBW);
reportPrintWriter.println("NB.; SCOPE; CKAN_CONNECTOR_URL;");
errorPrintWriter = new PrintWriter(errorBW);
errorPrintWriter.println("NB.; SCOPE; ERROR MESSAGE;");
} catch (Exception e) {
e.printStackTrace();
}
// final Path destination = Paths.get("report_matching_gn_catalogue.csv");
// Files.w(re, destination);
List<String> scopesProd = new ArrayList<String>();
// RuntimeResourceReader readerRR;
try {
// MODE-1
// readerRR = new RuntimeResourceReader(rootScope, platformName, null);
// System.out.println(readerRR.toString());
// System.out.println("Using GN: "+readerRR.getParameters());
// System.out.println("Scopes are: "+readerRR.getScopes().size());
// scopesProd.addAll(readerRR.getScopes());
// MODE-2
Map<String, String> mapScopes = GetAllInfrastructureScopes.loadMapOfScopeNameToFullScope(rootScope);
scopesProd.addAll(mapScopes.values());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Read " + scopesProd.size() + " VREs from root scope: " + rootScope);
for (String vre : scopesProd) {
System.out.println("VRE: " + vre);
}
int count = 0;
try {
final String ckanResource = "org.gcube.data.access.ckanconnector.CkanConnector";
final String serviceName = "CkanConnector";
final String serviceClass = "DataAccess";
int totalScope = scopesProd.size();
int reportOp = 0;
int errorOp = 0;
for (String scope : scopesProd) {
count++;
System.out.println("#### Operation "+ count+ " of "+totalScope+") Fetching scope: " + scope);
ScopeProvider.instance.set(scope);
GcoreEndpointReader reader = null;
try {
// discovery = new GCatClientDiscovery();
reader = new GcoreEndpointReader(scope, serviceClass, serviceName, ckanResource);
} catch (Exception e) {
System.out.println("gCat not instanciable in the scope: " + scope);
errorOp++;
writeError(errorOp + "; " + scope + " ; " + e.getMessage());
}
if (reader != null) {
reportOp++;
writeReport(reportOp + "; " + scope + " ; " + reader.getEndPoints().get(0) + " ; ");
}
System.out.println("Sleeping...");
Thread.sleep(500);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (reportPrintWriter != null)
reportPrintWriter.close();
if (errorPrintWriter != null)
errorPrintWriter.close();
System.out.println("Performed fetching from " + count + " scopes");
System.out.println("FINISHED!!!");
}
}
/**
* Write report.
*
* @param newline the newline
*/
private static synchronized void writeReport(String newline) {
c++;
reportPrintWriter.println(newline);
}
/**
* Write error.
*
* @param newline the newline
*/
private static synchronized void writeError(String newline) {
c++;
errorPrintWriter.println(newline);
}
}

View File

@ -1,23 +1,10 @@
package org.gcube.datatransfer.test; package org.gcube.datatransfer.test;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.storagehub.client.StreamDescriptor; import org.gcube.common.storagehub.client.StreamDescriptor;
import org.gcube.common.storagehub.client.plugins.AbstractPlugin; import org.gcube.common.storagehub.client.plugins.AbstractPlugin;
import org.gcube.common.storagehub.client.proxies.ItemManagerClient; import org.gcube.common.storagehub.client.proxies.ItemManagerClient;
import org.gcube.datatransfer.resolver.catalogue.resource.CkanCatalogueConfigurationsReader;
import org.gcube.datatransfer.resolver.catalogue.resource.GatewayCKANCatalogueReference;
import org.gcube.datatransfer.resolver.init.UriResolverSmartGearManagerInit; import org.gcube.datatransfer.resolver.init.UriResolverSmartGearManagerInit;
import org.gcube.datatransfer.resolver.services.CatalogueResolver;
import org.gcube.datatransfer.resolver.services.StorageHubResolver;
import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.datatransfer.resolver.shub.StorageHubMetadataResponseBuilder;
import org.junit.Before;
import org.junit.Test;
import com.itextpdf.text.log.SysoCounter;
/** /**
@ -34,7 +21,7 @@ public class StorageHubTest {
private static String vreName = "BlueBridgeProject"; private static String vreName = "BlueBridgeProject";
private String rootContextScope = "/d4science.research-infrastructures.eu"; private String rootContextScope = "/d4science.research-infrastructures.eu";
private String authorizationToken = "ea16e0fa-722a-4589-83b0-0a731d2d4039-843339462"; private String authorizationToken = "";
//@Before //@Before
public void init() { public void init() {

View File

@ -0,0 +1,161 @@
package resources;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class GcoreEndpointReader.
*
* @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it
* Jun 10, 2016
*/
public class GcoreEndpointReader implements Serializable{
private static final long serialVersionUID = 7631710353375893823L;
private static final Logger logger = LoggerFactory.getLogger(GcoreEndpointReader.class);
private List<String> endPoints;
/**
* Instantiates a new gcore endpoint reader.
*
* @param scope the scope
* @throws Exception the exception
*/
public GcoreEndpointReader(String scope, String serviceClass, String serviceName, String ckanResource) throws Exception {
String currentScope = ScopeProvider.instance.get();
try{
logger.info("set scope "+scope);
ScopeProvider.instance.set(scope);
SimpleQuery query = queryFor(GCoreEndpoint.class);
query.addCondition(String.format("$resource/Profile/ServiceClass/text() eq '%s'",serviceClass));
query.addCondition("$resource/Profile/DeploymentData/Status/text() eq 'ready'");
query.addCondition(String.format("$resource/Profile/ServiceName/text() eq '%s'",serviceName));
query.addCondition(String.format("$resource/Scopes/Scope/text()[.='%s']", scope)); // i.e. check the resource contains among the scopes this one
query.setResult("$resource/Profile/AccessPoint/RunningInstanceInterfaces//Endpoint[@EntryName/string() eq \""+ckanResource+"\"]/text()");
logger.debug("submitting quey "+query.toString());
DiscoveryClient<String> client = client();
List<String> endpoints = client.submit(query);
if (endpoints == null || endpoints.isEmpty()) throw new Exception("Cannot retrieve the GCoreEndpoint serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope);
for (String endpoint : endpoints) {
System.out.println("end point "+endpoint);
}
this.endPoints = endpoints;
if(endPoints==null)
throw new Exception("Endpoint:"+ckanResource+", is null for serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope);
logger.info("found entyname "+endPoints+" for ckanResource: "+ckanResource);
/*Group<Endpoint> accessPoints = se.profile().endpoints();
if(accessPoints.size()==0) throw new Exception("Endpoint in serviceName serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope +" not found");
Endpoint ep = accessPoints.iterator().next();
String epName = ep.name();
System.out.println(epName);*/
}catch(Exception e){
String error = "An error occurred during GCoreEndpoint discovery, serviceName: "+serviceName +", serviceClass: " +serviceClass +", in scope: "+scope +".";
logger.error(error, e);
throw new Exception(error);
}finally{
logger.info("scope provider reset");
ScopeProvider.instance.set(currentScope);
}
}
/**
* Gets the end points.
*
* @return the end points
*/
public List<String> getEndPoints() {
return endPoints;
}
// private static String[] scopes = {"" +
// "/d4science.research-infrastructures.eu/gCubeApps",
// "/d4science.research-infrastructures.eu/FARM",
// "/d4science.research-infrastructures.eu/D4Research",
// "/d4science.research-infrastructures.eu/OpenAIRE",
// "/d4science.research-infrastructures.eu/Edison",
// "/d4science.research-infrastructures.eu/SmartArea",
// "/d4science.research-infrastructures.eu/SoBigData"};
static PrintWriter reportPrintWriter;
static PrintWriter errorPrintWriter;
/**
* The main method.
*
* @param args the arguments
*/
public static void main(String[] args) {
FileWriter reportWriter = null;
final String ckanResource = "org.gcube.data.access.ckanconnector.CkanConnector";
final String serviceName = "CkanConnector";
final String serviceClass = "DataAccess";
final String rootScope = "/d4science.research-infrastructures.eu";
try{
reportWriter = new FileWriter("report_ckanconnector.csv", true);
BufferedWriter reportBW = new BufferedWriter(reportWriter);
reportPrintWriter = new PrintWriter(reportBW);
reportPrintWriter.println("NB.; SCOPE; END POINTS;");
try {
int i = 0;
Map<String, String> scopes = GetAllInfrastructureScopes.loadMapOfScopeNameToFullScope(rootScope);
for (String scope : scopes.values()) {
System.out.println("Ckecking scope: "+scope);
ScopeProvider.instance.set(scope);
try{
i++;
GcoreEndpointReader reader = new GcoreEndpointReader(scope, serviceClass, serviceName, ckanResource);
System.out.println("Appending endpoints: "+reader.getEndPoints());
reportPrintWriter.println(i+"; "+scope+"; "+reader.getEndPoints());
}catch(Exception e){
i--;
e.printStackTrace();
}
}
}
catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}catch(Exception e){
e.printStackTrace();
}
finally{
if(reportPrintWriter!=null)
reportPrintWriter.close();
}
}
}

View File

@ -0,0 +1,221 @@
package resources;
import static org.gcube.resources.discovery.icclient.ICFactory.client;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.gcube.common.resources.gcore.utils.XPathHelper;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.Query;
import org.gcube.resources.discovery.client.queries.impl.QueryBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
/**
* The Class GetAllInfrastructureScopes.
*
* @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it)
*
* May 16, 2019
*/
public class GetAllInfrastructureScopes {
public static Logger logger = LoggerFactory.getLogger(GetAllInfrastructureScopes.class);
protected static final String RESOURCE_PROFILE_NAME_TEXT = "/Resource/Profile/Name/text()";
/**
* Load map of scope name to full scope.
*
* @param rootScope the root scope
* @return the map of binding between (VRE_NAME, FULL_SCOPE_OF_VRE_NAME)
* @throws Exception the exception
*/
public static Map<String, String> loadMapOfScopeNameToFullScope(String rootScope) throws Exception{
String originalScope = ScopeProvider.instance.get();
try{
ScopeBean scopeBean = null;
Map<String, String> scopeNameToFullScopeMap = new HashMap<String,String>();
ScopeProvider.instance.set(rootScope);
String secondaryType = Type.INFRASTRUCTURE.name();
scopeBean = new ScopeBean(rootScope);
logger.info("Added the couple ({},{}) as {} (NAME, FULL SCOPE) into map", scopeBean.name(), rootScope, secondaryType);
scopeNameToFullScopeMap.put(scopeBean.name(), rootScope);
List<String> listVOScopes = getListOfVOScopes(secondaryType);
logger.info("Searching for secondaryType={} the scope/s found is/are: " +secondaryType, listVOScopes);
//int noVOTypeCount = 0;
for (String voScope : listVOScopes) {
//int count = voScope.length() - voScope.replace("/", "").length();
scopeBean = new ScopeBean(voScope);
//IS A VO
//if(count==2){
if(scopeBean.is(Type.VO)){
secondaryType = Type.VO.name();
logger.info("{} is a {}...",voScope,secondaryType);
ScopeProvider.instance.set(voScope);
scopeBean = new ScopeBean(voScope);
logger.info("Added the couple ({},{}) as {} (NAME, FULL SCOPE) into map", scopeBean.name(), voScope, secondaryType);
scopeNameToFullScopeMap.put(scopeBean.name(), voScope);
secondaryType = Type.VRE.name();
List<String> listVREs = getListOfResourcesForSecondaryType(secondaryType);
logger.debug("VREs found for VO "+voScope+ " is/are "+listVREs.size()+ ": "+listVREs);
for (String vreName : listVREs) {
String vreScope = String.format("%s/%s", voScope,vreName);
scopeNameToFullScopeMap.put(vreName, vreScope);
}
}else{
//noVOTypeCount++;
logger.info(voScope +" is not a VO, skipping it");
}
}
/*System.out.println("Total VO is: "+(listVOScopes.size()+noVOTypeCount));
for (String vreName : vreNameFullScope.keySet()) {
System.out.println("VRE Name: "+vreName + " has scope: "+vreNameFullScope.get(vreName));
}*/
logger.info("Total VRE is: "+scopeNameToFullScopeMap.size());
return scopeNameToFullScopeMap;
}catch(Exception e ){
throw new Exception("Error on loading the map of VRE nameto scope: ", e);
}
finally{
if(originalScope!=null && !originalScope.isEmpty()){
ScopeProvider.instance.set(originalScope);
}else
ScopeProvider.instance.reset();
}
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
private static List<String> getListOfResourcesForSecondaryType(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listResourceName = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
if(resourceNames!=null && resourceNames.size()>0)
listResourceName.add(resourceNames.get(0));
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listResourceName;
}
/**
* Gets the list of resources for secondary type.
*
* @param secondaryType the secondary type
* @return the list of resource names for the input secondary type
*/
private static List<String> getListOfVOScopes(String secondaryType) {
String queryString = "for $profile in collection('/db/Profiles/GenericResource')//Resource " +
"where $profile/Profile/SecondaryType/string() eq '"+secondaryType+"' return $profile";
List<String> listOfVOScopes = new ArrayList<String>();
try {
logger.info("Trying to fetch GenericResource in the scope: "+ScopeProvider.instance.get()+", SecondaryType: " + secondaryType);
Query q = new QueryBox(queryString);
DiscoveryClient<String> client = client();
List<String> listGenericResources = client.submit(q);
logger.info("# of GenericResource returned searching for secondaryType= "+secondaryType+" is/are: "+listGenericResources.size());
if (listGenericResources == null || listGenericResources.size() == 0)
throw new Exception("GenericResource with SecondaryType: " + secondaryType + ", is not registered in the scope: "+ScopeProvider.instance.get());
else {
for (String genericResource : listGenericResources) {
try{
String elem = genericResource;
DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document document = docBuilder.parse(new InputSource(new StringReader(elem)));
Element rootElement = document.getDocumentElement();
XPathHelper helper = new XPathHelper(rootElement);
// List<String> resourceNames = helper.evaluate(RESOURCE_PROFILE_NAME_TEXT);
//
// if(resourceNames!=null && resourceNames.size()>0)
// listResourceName.add(resourceNames.get(0));
List<String> scopes = helper.evaluate("/Resource/Profile/Body/infrastructures/infrastructure/vos/vo/scope/text()");
for (String scopeFound : scopes) {
listOfVOScopes.add(scopeFound);
}
}catch(Exception e){
throw new Exception("Error during parsing the generic resource: "+genericResource + " in the scope: "+ScopeProvider.instance.get());
}
}
}
} catch (Exception e) {
logger.error("Error while trying to fetch generic resource from the infrastructure", e);
}
return listOfVOScopes;
}
}

View File

@ -15,19 +15,10 @@ import java.nio.file.StandardCopyOption;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.http.HttpStatus; import org.apache.http.HttpStatus;
import org.gcube.common.storagehub.client.StreamDescriptor;
import org.gcube.common.storagehub.client.plugins.AbstractPlugin;
import org.gcube.common.storagehub.client.proxies.ItemManagerClient;
import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileReader; import org.gcube.datatransfer.resolver.applicationprofile.ApplicationProfileReader;
import org.gcube.datatransfer.resolver.services.StorageHubResolver;
import org.gcube.datatransfer.resolver.services.error.ExceptionManager;
import org.gcube.datatransfer.resolver.shub.StorageHubMetadataResponseBuilder;
import org.gcube.datatransfer.resolver.util.HTTPCallsUtils; import org.gcube.datatransfer.resolver.util.HTTPCallsUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -42,7 +33,7 @@ import org.slf4j.LoggerFactory;
public class TestResolvers { public class TestResolvers {
public static final String URI_RESOLVER_SERVICE_ENDPOINT = "https://data1-d.d4science.net"; public static final String URI_RESOLVER_SERVICE_ENDPOINT = "https://data.dev.d4science.org";
public static final Logger logger = LoggerFactory.getLogger(TestResolvers.class); public static final Logger logger = LoggerFactory.getLogger(TestResolvers.class);