gCubeSystem
/
dataminer
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Managed Authorization 2.0 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@131858 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2016-09-27 13:44:14 +00:00
parent 3f5287800f
commit 3c2f098f55
9 changed files with 187 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/capabilities/GetCapabilitiesBuilder.java
View File

@ -61,8 +61,10 @@ public class GetCapabilitiesBuilder {
tokenm.getCredentials(); tokenm.getCredentials();
String scope = tokenm.getScope(); String scope = tokenm.getScope();
String username = tokenm.getUserName(); String username = tokenm.getUserName();
String token = tokenm.getToken();
basicInputs.put(ConfigurationManager.scopeParameter, scope); basicInputs.put(ConfigurationManager.scopeParameter, scope);
basicInputs.put(ConfigurationManager.usernameParameter, username); basicInputs.put(ConfigurationManager.usernameParameter, username);
basicInputs.put(ConfigurationManager.tokenParameter, token);
configManager.configAlgorithmEnvironment(basicInputs); configManager.configAlgorithmEnvironment(basicInputs);
AnalysisLogger.getLogger().debug("Initializing Capabilities Skeleton in scope " + configManager.getScope() + " with user " + configManager.getUsername()); AnalysisLogger.getLogger().debug("Initializing Capabilities Skeleton in scope " + configManager.getScope() + " with user " + configManager.getUsername());

85
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/capabilities/GetCapabilitiesChecker.java
View File

@ -39,8 +39,6 @@ public class GetCapabilitiesChecker {
URL scannedUrl = Thread.currentThread().getContextClassLoader().getResource(scannedPath); URL scannedUrl = Thread.currentThread().getContextClassLoader().getResource(scannedPath);
String jarPath = scannedUrl.getFile(); String jarPath = scannedUrl.getFile();
AnalysisLogger.getLogger().debug("Jar Path complete: " + jarPath); AnalysisLogger.getLogger().debug("Jar Path complete: " + jarPath);
jarPath = jarPath.substring(jarPath.indexOf("file:/") + 6, jarPath.lastIndexOf("!")); jarPath = jarPath.substring(jarPath.indexOf("file:/") + 6, jarPath.lastIndexOf("!"));
if (jarPath.startsWith("home")) if (jarPath.startsWith("home"))
@ -50,11 +48,23 @@ public class GetCapabilitiesChecker {
JarFile jarFile = null; JarFile jarFile = null;
List<Class<?>> result = new ArrayList<Class<?>>(); List<Class<?>> result = new ArrayList<Class<?>>();
File otherjar = new File(new File(jarPath).getParent(),"dataminer-algorithms.jar"); String pathTojars = new File(jarPath).getParent();
if (otherjar .exists())
jarPath =otherjar.getAbsolutePath(); File[] jars = new File(pathTojars).listFiles();
AnalysisLogger.getLogger().debug("Alternative Jar Path: " + jarPath);
try { try {
for (File jar : jars) {
// File otherjar = new File(new File(jarPath).getParent(),"dataminer-algorithms.jar");
if (jar.getName().equals("dataminer-algorithms.jar") || jar.getName().endsWith("_interface.jar")) {
//File otherjar = new File(new File(jarPath).getParent(), "dataminer-algorithms.jar");
File otherjar = jar;
if (otherjar.exists())
jarPath = otherjar.getAbsolutePath();
AnalysisLogger.getLogger().debug("Alternative Jar Path: " + jarPath);
jarFile = new JarFile(jarPath); jarFile = new JarFile(jarPath);
Enumeration<JarEntry> en = jarFile.entries(); Enumeration<JarEntry> en = jarFile.entries();
@ -74,8 +84,10 @@ public class GetCapabilitiesChecker {
// do nothing, just continue processing classes // do nothing, just continue processing classes
} }
} }
} }// while
}// if jar known
}
return result; return result;
} catch (Exception e) { } catch (Exception e) {
throw e; throw e;
@ -167,29 +179,22 @@ public class GetCapabilitiesChecker {
} }
} }
public static Charset getConnectionCharset(URLConnection connection) { public static Charset getConnectionCharset(URLConnection connection) {
String contentType = null; String contentType = null;
try { try {
contentType = connection.getContentType(); contentType = connection.getContentType();
} } catch (Exception e) {
catch (Exception e) {
// specified charset is not found, // specified charset is not found,
// skip it to return the default one // skip it to return the default one
return Charset.defaultCharset(); return Charset.defaultCharset();
} }
if (contentType != null && contentType.length() > 0) if (contentType != null && contentType.length() > 0) {
{
contentType = contentType.toLowerCase(); contentType = contentType.toLowerCase();
String charsetName = extractCharsetName(contentType); String charsetName = extractCharsetName(contentType);
if (charsetName != null && charsetName.length() > 0) if (charsetName != null && charsetName.length() > 0) {
{ try {
try
{
return Charset.forName(charsetName); return Charset.forName(charsetName);
} } catch (Exception e) {
catch (Exception e) {
// specified charset is not found, // specified charset is not found,
// skip it to return the default one // skip it to return the default one
} }
@ -199,27 +204,25 @@ public class GetCapabilitiesChecker {
// return the default charset // return the default charset
return Charset.defaultCharset(); return Charset.defaultCharset();
} }
/** /**
* Extract the charset name form the content type string. * Extract the charset name form the content type string. Content type string is received from Content-Type header.
* Content type string is received from Content-Type header.
* *
* @param contentType the content type string, must be not null. * @param contentType
* the content type string, must be not null.
* @return the found charset name or null if not found. * @return the found charset name or null if not found.
*/ */
private static String extractCharsetName(String contentType) { private static String extractCharsetName(String contentType) {
// split onto media types // split onto media types
final String[] mediaTypes = contentType.split(":"); final String[] mediaTypes = contentType.split(":");
if (mediaTypes.length > 0) if (mediaTypes.length > 0) {
{
// use only the first one, and split it on parameters // use only the first one, and split it on parameters
final String[] params = mediaTypes[0].split(";"); final String[] params = mediaTypes[0].split(";");
// find the charset parameter and return it's value // find the charset parameter and return it's value
for (String each : params) for (String each : params) {
{
each = each.trim(); each = each.trim();
if (each.startsWith("charset=")) if (each.startsWith("charset=")) {
{
// return the charset name // return the charset name
return each.substring(8).trim(); return each.substring(8).trim();
} }
@ -229,18 +232,17 @@ public class GetCapabilitiesChecker {
return null; return null;
} }
private static String RUNTIME_RESOURCE_NAME = "ReportsStoreGateway"; private static String RUNTIME_RESOURCE_NAME = "ReportsStoreGateway";
private static String CATEGORY_NAME = "Service"; private static String CATEGORY_NAME = "Service";
public static String readPageNoHttpClient(URL url) throws Exception{ public static String readPageNoHttpClient(URL url) throws Exception {
URLConnection conn = url.openConnection(); URLConnection conn = url.openConnection();
//pretend you're a browser (make my request from Java more browsery-like.) // pretend you're a browser (make my request from Java more browsery-like.)
conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
conn.setDoOutput(true); conn.setDoOutput(true);
conn.setAllowUserInteraction(true); conn.setAllowUserInteraction(true);
conn.setConnectTimeout(25*60000); conn.setConnectTimeout(25 * 60000);
conn.setReadTimeout(25*60000); conn.setReadTimeout(25 * 60000);
Charset charset = getConnectionCharset(conn); Charset charset = getConnectionCharset(conn);
@ -259,15 +261,14 @@ public class GetCapabilitiesChecker {
return page; return page;
} }
public static String readPageHTTPHeader(URL url, String token) throws Exception {
public static String readPageHTTPHeader(URL url,String token) throws Exception{
URLConnection conn = url.openConnection(); URLConnection conn = url.openConnection();
//pretend you're a browser (make my request from Java more browsery-like.) // pretend you're a browser (make my request from Java more browsery-like.)
conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
conn.setDoOutput(true); conn.setDoOutput(true);
conn.setAllowUserInteraction(true); conn.setAllowUserInteraction(true);
conn.setConnectTimeout(25*60000); conn.setConnectTimeout(25 * 60000);
conn.setReadTimeout(25*60000); conn.setReadTimeout(25 * 60000);
conn.setRequestProperty("gcube-token", token); conn.setRequestProperty("gcube-token", token);
Charset charset = getConnectionCharset(conn); Charset charset = getConnectionCharset(conn);
@ -285,9 +286,9 @@ public class GetCapabilitiesChecker {
System.out.println(page); System.out.println(page);
conn.getInputStream().close(); conn.getInputStream().close();
return page; return page;
} }
//build config.xml // build config.xml
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
String packageS = "org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses"; String packageS = "org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses";
List<Class<?>> classes = GetCapabilitiesChecker.find(packageS); List<Class<?>> classes = GetCapabilitiesChecker.find(packageS);
@ -341,8 +342,4 @@ public class GetCapabilitiesChecker {
} }
} }

2
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java
View File

@ -18,7 +18,7 @@ public class InfrastructureDialoguer {
public String scope; public String scope;
public InfrastructureDialoguer(String scope){ public InfrastructureDialoguer(String scope){
ScopeProvider.instance.set(scope); //ScopeProvider.instance.set(scope);
this.scope = scope; this.scope = scope;
} }

32
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java
View File

@ -314,19 +314,40 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm {
updateComputationOnWS(status, exception, null, null); updateComputationOnWS(status, exception, null, null);
} }
public void updateComputationOnWS(float status, String exception, List<StoredData> inputData, List<File> generatedData) { class RunDataspaceManager implements Runnable{
if (currentComputation != null) { List<StoredData> inputData;
currentComputation.setStatus("" + status); List<File> generatedData;
if (exception != null && exception.length() > 0) public RunDataspaceManager(List<StoredData> inputData, List<File> generatedData){
currentComputation.setException(exception); this.inputData=inputData;
this.generatedData=generatedData;
}
public void run() {
DataspaceManager manager = new DataspaceManager(config, currentComputation, inputData, null, generatedData); DataspaceManager manager = new DataspaceManager(config, currentComputation, inputData, null, generatedData);
try { try {
AnalysisLogger.getLogger().debug("Dataspace->Status updater->Writing computational info on the WS asyncronously");
manager.writeRunningComputationData(); manager.writeRunningComputationData();
} catch (Exception ez) { } catch (Exception ez) {
AnalysisLogger.getLogger().debug("Dataspace->Status updater->Impossible to write computation information on the Workspace"); AnalysisLogger.getLogger().debug("Dataspace->Status updater->Impossible to write computation information on the Workspace");
AnalysisLogger.getLogger().debug(ez); AnalysisLogger.getLogger().debug(ez);
} }
} }
};
public void updateComputationOnWS(float status, String exception, List<StoredData> inputData, List<File> generatedData) {
if (currentComputation != null) {
currentComputation.setStatus("" + status);
if (exception != null && exception.length() > 0)
currentComputation.setException(exception);
RunDataspaceManager rundm = new RunDataspaceManager(inputData,generatedData);
rundm.run();
/*
Thread t = new Thread(rundm);
t.start();
*/
}
} }
@Execute @Execute
@ -533,6 +554,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm {
ComputationData computation = new ComputationData(config.getTaskID(), config.getAgent(), agent.getDescription(), agent.getInfrastructure().name(), startTime, endTime, "100", config.getTaskID(), config.getParam(ConfigurationManager.serviceUserNameParameterVariable), config.getGcubeScope(), this.getClass().getCanonicalName()); ComputationData computation = new ComputationData(config.getTaskID(), config.getAgent(), agent.getDescription(), agent.getInfrastructure().name(), startTime, endTime, "100", config.getTaskID(), config.getParam(ConfigurationManager.serviceUserNameParameterVariable), config.getGcubeScope(), this.getClass().getCanonicalName());
// post on WS // post on WS
DataspaceManager manager = new DataspaceManager(config, computation, inputData, outputData, generatedFiles); DataspaceManager manager = new DataspaceManager(config, computation, inputData, outputData, generatedFiles);
Thread t = new Thread(manager); Thread t = new Thread(manager);
t.start(); t.start();
AnalysisLogger.getLogger().debug("Provenance manager running"); AnalysisLogger.getLogger().debug("Provenance manager running");

11
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java
View File

@ -19,6 +19,7 @@ public class ConfigurationManager {
public static String webPersistencePathVariable = ""; public static String webPersistencePathVariable = "";
public static String usernameParameter = "user.name"; public static String usernameParameter = "user.name";
public static String scopeParameter = "scope"; public static String scopeParameter = "scope";
public static String tokenParameter = "usertoken";
public static String defaultScope= "/gcube/devsec"; public static String defaultScope= "/gcube/devsec";
public static String defaultUsername= "statistical.wps"; public static String defaultUsername= "statistical.wps";
@ -57,6 +58,7 @@ public class ConfigurationManager {
private AlgorithmConfiguration config; private AlgorithmConfiguration config;
private String scope; private String scope;
private String username; private String username;
private String token;
public String getScope() { public String getScope() {
return scope; return scope;
@ -122,11 +124,20 @@ public class ConfigurationManager {
scope = "/" + scope; scope = "/" + scope;
username = (String) inputs.get(usernameParameter); username = (String) inputs.get(usernameParameter);
token = (String) inputs.get(tokenParameter);
AnalysisLogger.getLogger().debug("User name used by the client: " + username); AnalysisLogger.getLogger().debug("User name used by the client: " + username);
AnalysisLogger.getLogger().debug("User token used by the client: " + token);
if (username == null || username.trim().length() == 0) if (username == null || username.trim().length() == 0)
throw new Exception("Error: user name parameter (user.name) not set! This violates e-Infrastructure security policies"); throw new Exception("Error: user name parameter (user.name) not set! This violates e-Infrastructure security policies");
if (token == null || token.trim().length() == 0)
throw new Exception("Error: token parameter not set! This violates e-Infrastructure security policies");
config.setGcubeScope(scope); config.setGcubeScope(scope);
config.setGcubeUserName(username);
config.setGcubeToken(token);
// DONE get username from request // DONE get username from request
config.setParam(serviceUserNameParameterVariable, username); config.setParam(serviceUserNameParameterVariable, username);
config.setParam(processingSessionVariable, "" + UUID.randomUUID()); config.setParam(processingSessionVariable, "" + UUID.randomUUID());

26
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java
View File

@ -1,15 +1,18 @@
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping; package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping;
import static org.gcube.common.authorization.client.Constants.authorizationService;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.ClientInfo; import org.gcube.common.authorization.library.provider.ClientInfo;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.provider.UserInfo; import org.gcube.common.authorization.library.provider.UserInfo;
import org.gcube.common.scope.api.ScopeProvider;
public class TokenManager { public class TokenManager {
String username; String username;
String scope; String scope;
String token;
public String getScope(){ public String getScope(){
return scope; return scope;
@ -19,26 +22,35 @@ public class TokenManager {
return username; return username;
} }
public String getToken(){
return token;
}
public void getCredentials() { public void getCredentials() {
try{ try{
System.out.println("Retrieving token credentials"); System.out.println("Retrieving token credentials");
scope = ScopeProvider.instance.get(); //scope = ScopeProvider.instance.get();
System.out.println("Credentials from the GHN: scope: "+scope); System.out.println("Credentials from the GHN: scope: "+scope);
//get username from SmartGears //get username from SmartGears
ClientInfo token = (UserInfo) AuthorizationProvider.instance.get().getClient(); ClientInfo tokenInfo = (UserInfo) AuthorizationProvider.instance.get().getClient();
username = token.getId(); username = tokenInfo.getId();
System.out.println("Credentials from the GHN: user: "+username); token = SecurityTokenProvider.instance.get();
AuthorizationEntry entry = authorizationService().get(token);
scope = entry.getContext();
System.out.println("Credentials from the GHN: user: "+username+" , "+scope+" , "+token);
}catch(Exception e){ }catch(Exception e){
e.printStackTrace(); e.printStackTrace();
System.out.println("Error Retrieving token credentials: "+e.getLocalizedMessage()); System.out.println("Error Retrieving token credentials: "+e.getLocalizedMessage());
scope = null; scope = null;
username= null; username= null;
} }
if ((scope==null || username==null) && ConfigurationManager.isSimulationMode()){ if ((scope==null || username==null) && ConfigurationManager.isSimulationMode()){
scope = ConfigurationManager.defaultScope; scope = ConfigurationManager.defaultScope;
username = ConfigurationManager.defaultUsername; username = ConfigurationManager.defaultUsername;
} }
System.out.println("Retrieved scope: "+scope+" Username: "+username +" SIMULATION MODE: "+ConfigurationManager.isSimulationMode()); System.out.println("Retrieved scope: "+scope+" Username: "+username +" Token "+token+" SIMULATION MODE: "+ConfigurationManager.isSimulationMode());
} }

6
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -176,7 +176,7 @@ public class DataspaceManager implements Runnable {
//fileItem.getProperties().addProperties(properties); //fileItem.getProperties().addProperties(properties);
AnalysisLogger.getLogger().debug("Dataspace->WS OP file saved on the WS " + filenameonwsString); AnalysisLogger.getLogger().debug("Dataspace->WS OP file saved on the WS " + filenameonwsString);
url = fileItem.getPublicLink(true); url = fileItem.getPublicLink(false);
AnalysisLogger.getLogger().debug("Dataspace->WS OP url produced for the file " + url); AnalysisLogger.getLogger().debug("Dataspace->WS OP url produced for the file " + url);
data.payload = url; data.payload = url;
@ -184,7 +184,7 @@ public class DataspaceManager implements Runnable {
in.close(); in.close();
} catch (Exception e) { } catch (Exception e) {
AnalysisLogger.getLogger().debug("Dataspace->Error creating file " + e.getMessage()); AnalysisLogger.getLogger().debug("Dataspace->Error creating file " + e.getMessage());
AnalysisLogger.getLogger().debug(e); //AnalysisLogger.getLogger().debug(e);
} }
AnalysisLogger.getLogger().debug("Dataspace->File created " + filenameonwsString); AnalysisLogger.getLogger().debug("Dataspace->File created " + filenameonwsString);
} else { } else {
@ -194,7 +194,7 @@ public class DataspaceManager implements Runnable {
} catch (Throwable e) { } catch (Throwable e) {
e.printStackTrace(); e.printStackTrace();
AnalysisLogger.getLogger().debug("Dataspace->Could not retrieve input payload " + data.payload+" - "+e.getLocalizedMessage()); AnalysisLogger.getLogger().debug("Dataspace->Could not retrieve input payload " + data.payload+" - "+e.getLocalizedMessage());
AnalysisLogger.getLogger().debug(e); //AnalysisLogger.getLogger().debug(e);
url = "payload was not made available for this dataset"; url = "payload was not made available for this dataset";
data.payload = url; data.payload = url;
} }

35
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/CreateTestTableFromFile.java Normal file
View File

@ -0,0 +1,35 @@
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.test;
import java.io.File;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure.DatabaseInfo;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure.InfrastructureDialoguer;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping.InputsManager;
public class CreateTestTableFromFile {
public static void main(String[] args) throws Exception{
Regressor regressor = new Regressor();
AlgorithmConfiguration config = regressor.getConfig();
String tableName = "spread_test";
File tableFile = new File("C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\spread_input_dataset .csv");
String inputTableTemplate = TableTemplates.GENERIC.name();
String scope = "/gcube/devsec/devVRE";
System.out.println("Asking the infra for database in scope: "+scope);
InfrastructureDialoguer dialoguer = new InfrastructureDialoguer(scope);
DatabaseInfo supportDatabaseInfo = dialoguer.getDatabaseInfo("StatisticalManagerDataBase");
InputsManager manager = new InputsManager(null, config, "test");
manager.configSupportDatabaseParameters(supportDatabaseInfo);
System.out.println("Database retrieved: "+supportDatabaseInfo.url);
System.out.println("Creating: "+tableName);
manager.createTable(tableName, tableFile,manager.getConfig(), supportDatabaseInfo, inputTableTemplate);
System.out.println("DB coordinates: \nURL "+supportDatabaseInfo.url+"\n user "+supportDatabaseInfo.username+"\n password "+supportDatabaseInfo.password+"\n driver "+supportDatabaseInfo.driver+"\n dialect "+supportDatabaseInfo.dialect);
System.out.println("All done");
}
}

1
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/web/ExecuteRequest.java
View File

@ -795,6 +795,7 @@ public class ExecuteRequest extends Request implements IObserver {
try { try {
is = executeDocument.newInputStream(); is = executeDocument.newInputStream();
DatabaseFactory.getDatabase().insertRequest(wpsid, is, true); DatabaseFactory.getDatabase().insertRequest(wpsid, is, true);
//DatabaseFactory.getDatabase().insertRequest(wpsid, is, false);
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("Exception storing ExecuteRequest", e); LOGGER.error("Exception storing ExecuteRequest", e);
} finally { } finally {