gCubeSystem
/
dataminer
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Managed Authorization 2.0 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/DataMiner@131858 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2016-09-27 13:44:14 +00:00
parent 3f5287800f
commit 3c2f098f55
9 changed files with 187 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/capabilities/GetCapabilitiesBuilder.java
View File

@ -61,8 +61,10 @@ public class GetCapabilitiesBuilder {
tokenm.getCredentials();
String scope = tokenm.getScope();
String username = tokenm.getUserName();
String token = tokenm.getToken();
basicInputs.put(ConfigurationManager.scopeParameter, scope);
basicInputs.put(ConfigurationManager.usernameParameter, username);
basicInputs.put(ConfigurationManager.tokenParameter, token);
configManager.configAlgorithmEnvironment(basicInputs);
AnalysisLogger.getLogger().debug("Initializing Capabilities Skeleton in scope " + configManager.getScope() + " with user " + configManager.getUsername());

179
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/capabilities/GetCapabilitiesChecker.java
View File

@ -38,9 +38,7 @@ public class GetCapabilitiesChecker {
String scannedPath = packageName.replace(".", "/");
URL scannedUrl = Thread.currentThread().getContextClassLoader().getResource(scannedPath);
String jarPath = scannedUrl.getFile();
AnalysisLogger.getLogger().debug("Jar Path complete: " + jarPath);
jarPath = jarPath.substring(jarPath.indexOf("file:/") + 6, jarPath.lastIndexOf("!"));
if (jarPath.startsWith("home"))
@ -49,33 +47,47 @@ public class GetCapabilitiesChecker {
JarFile jarFile = null;
List<Class<?>> result = new ArrayList<Class<?>>();
File otherjar = new File(new File(jarPath).getParent(),"dataminer-algorithms.jar");
if (otherjar .exists())
jarPath =otherjar.getAbsolutePath();
AnalysisLogger.getLogger().debug("Alternative Jar Path: " + jarPath);
String pathTojars = new File(jarPath).getParent();
File[] jars = new File(pathTojars).listFiles();
try {
jarFile = new JarFile(jarPath);
Enumeration<JarEntry> en = jarFile.entries();
while (en.hasMoreElements()) {
JarEntry entry = en.nextElement();
String entryName = entry.getName();
packageName = packageName.replace('.', '/');
for (File jar : jars) {
// File otherjar = new File(new File(jarPath).getParent(),"dataminer-algorithms.jar");
if (entryName != null && entryName.endsWith(".class") && entryName.startsWith(packageName)) {
try {
Class entryClass = Class.forName(entryName.substring(0, entryName.length() - 6).replace('/', '.'));
if (jar.getName().equals("dataminer-algorithms.jar") || jar.getName().endsWith("_interface.jar")) {
if (entryClass != null) {
result.add(entryClass);
//File otherjar = new File(new File(jarPath).getParent(), "dataminer-algorithms.jar");
File otherjar = jar;
if (otherjar.exists())
jarPath = otherjar.getAbsolutePath();
AnalysisLogger.getLogger().debug("Alternative Jar Path: " + jarPath);
jarFile = new JarFile(jarPath);
Enumeration<JarEntry> en = jarFile.entries();
while (en.hasMoreElements()) {
JarEntry entry = en.nextElement();
String entryName = entry.getName();
packageName = packageName.replace('.', '/');
if (entryName != null && entryName.endsWith(".class") && entryName.startsWith(packageName)) {
try {
Class entryClass = Class.forName(entryName.substring(0, entryName.length() - 6).replace('/', '.'));
if (entryClass != null) {
result.add(entryClass);
}
} catch (Throwable e) {
// do nothing, just continue processing classes
}
}
} catch (Throwable e) {
// do nothing, just continue processing classes
}
}
}
}// while
}// if jar known
}
return result;
} catch (Exception e) {
throw e;
@ -167,29 +179,22 @@ public class GetCapabilitiesChecker {
}
}
public static Charset getConnectionCharset(URLConnection connection) {
String contentType = null;
try {
contentType = connection.getContentType();
}
catch (Exception e) {
} catch (Exception e) {
// specified charset is not found,
// skip it to return the default one
return Charset.defaultCharset();
}
if (contentType != null && contentType.length() > 0)
{
if (contentType != null && contentType.length() > 0) {
contentType = contentType.toLowerCase();
String charsetName = extractCharsetName(contentType);
if (charsetName != null && charsetName.length() > 0)
{
try
{
if (charsetName != null && charsetName.length() > 0) {
try {
return Charset.forName(charsetName);
}
catch (Exception e) {
} catch (Exception e) {
// specified charset is not found,
// skip it to return the default one
}
@ -199,27 +204,25 @@ public class GetCapabilitiesChecker {
// return the default charset
return Charset.defaultCharset();
}
/**
* Extract the charset name form the content type string.
* Content type string is received from Content-Type header.
*
* @param contentType the content type string, must be not null.
* Extract the charset name form the content type string. Content type string is received from Content-Type header.
*
* @param contentType
* the content type string, must be not null.
* @return the found charset name or null if not found.
*/
private static String extractCharsetName(String contentType) {
// split onto media types
final String[] mediaTypes = contentType.split(":");
if (mediaTypes.length > 0)
{
if (mediaTypes.length > 0) {
// use only the first one, and split it on parameters
final String[] params = mediaTypes[0].split(";");
// find the charset parameter and return it's value
for (String each : params)
{
for (String each : params) {
each = each.trim();
if (each.startsWith("charset="))
{
if (each.startsWith("charset=")) {
// return the charset name
return each.substring(8).trim();
}
@ -229,65 +232,63 @@ public class GetCapabilitiesChecker {
return null;
}
private static String RUNTIME_RESOURCE_NAME = "ReportsStoreGateway";
private static String CATEGORY_NAME = "Service";
public static String readPageNoHttpClient(URL url) throws Exception{
URLConnection conn = url.openConnection();
//pretend you're a browser (make my request from Java more browsery-like.)
conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
conn.setDoOutput(true);
conn.setAllowUserInteraction(true);
conn.setConnectTimeout(25*60000);
conn.setReadTimeout(25*60000);
Charset charset = getConnectionCharset(conn);
BufferedReader dis = new BufferedReader(new InputStreamReader(conn.getInputStream(), charset));
String inputLine;
StringBuffer pageBuffer = new StringBuffer();
// Loop through each line, looking for the closing head element
while ((inputLine = dis.readLine()) != null) {
pageBuffer.append(inputLine + "\r\n");
}
String page = pageBuffer.toString();
System.out.println(page);
conn.getInputStream().close();
return page;
}
public static String readPageHTTPHeader(URL url,String token) throws Exception{
public static String readPageNoHttpClient(URL url) throws Exception {
URLConnection conn = url.openConnection();
//pretend you're a browser (make my request from Java more browsery-like.)
// pretend you're a browser (make my request from Java more browsery-like.)
conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
conn.setDoOutput(true);
conn.setAllowUserInteraction(true);
conn.setConnectTimeout(25*60000);
conn.setReadTimeout(25*60000);
conn.setRequestProperty("gcube-token", token);
conn.setConnectTimeout(25 * 60000);
conn.setReadTimeout(25 * 60000);
Charset charset = getConnectionCharset(conn);
BufferedReader dis = new BufferedReader(new InputStreamReader(conn.getInputStream(), charset));
String inputLine;
StringBuffer pageBuffer = new StringBuffer();
// Loop through each line, looking for the closing head element
while ((inputLine = dis.readLine()) != null) {
while ((inputLine = dis.readLine()) != null) {
pageBuffer.append(inputLine + "\r\n");
}
String page = pageBuffer.toString();
System.out.println(page);
conn.getInputStream().close();
return page;
}
//build config.xml
}
public static String readPageHTTPHeader(URL url, String token) throws Exception {
URLConnection conn = url.openConnection();
// pretend you're a browser (make my request from Java more browsery-like.)
conn.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
conn.setDoOutput(true);
conn.setAllowUserInteraction(true);
conn.setConnectTimeout(25 * 60000);
conn.setReadTimeout(25 * 60000);
conn.setRequestProperty("gcube-token", token);
Charset charset = getConnectionCharset(conn);
BufferedReader dis = new BufferedReader(new InputStreamReader(conn.getInputStream(), charset));
String inputLine;
StringBuffer pageBuffer = new StringBuffer();
// Loop through each line, looking for the closing head element
while ((inputLine = dis.readLine()) != null) {
pageBuffer.append(inputLine + "\r\n");
}
String page = pageBuffer.toString();
System.out.println(page);
conn.getInputStream().close();
return page;
}
// build config.xml
public static void main(String[] args) throws Exception {
String packageS = "org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses";
List<Class<?>> classes = GetCapabilitiesChecker.find(packageS);
@ -302,7 +303,7 @@ public class GetCapabilitiesChecker {
System.out.println(classes.size() + " algorithms");
}
public static void main1(String[] args) throws Exception {
String packageS = "org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses";
List<Class<?>> classes = GetCapabilitiesChecker.find(packageS);
@ -340,9 +341,5 @@ public class GetCapabilitiesChecker {
System.out.println("Checked " + counter + " algorithms");
}
}

2
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/infrastructure/InfrastructureDialoguer.java
View File

@ -18,7 +18,7 @@ public class InfrastructureDialoguer {
public String scope;
public InfrastructureDialoguer(String scope){
ScopeProvider.instance.set(scope);
//ScopeProvider.instance.set(scope);
this.scope = scope;
}

32
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/AbstractEcologicalEngineMapper.java
View File

@ -314,19 +314,40 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm {
updateComputationOnWS(status, exception, null, null);
}
public void updateComputationOnWS(float status, String exception, List<StoredData> inputData, List<File> generatedData) {
if (currentComputation != null) {
currentComputation.setStatus("" + status);
if (exception != null && exception.length() > 0)
currentComputation.setException(exception);
class RunDataspaceManager implements Runnable{
List<StoredData> inputData;
List<File> generatedData;
public RunDataspaceManager(List<StoredData> inputData, List<File> generatedData){
this.inputData=inputData;
this.generatedData=generatedData;
}
public void run() {
DataspaceManager manager = new DataspaceManager(config, currentComputation, inputData, null, generatedData);
try {
AnalysisLogger.getLogger().debug("Dataspace->Status updater->Writing computational info on the WS asyncronously");
manager.writeRunningComputationData();
} catch (Exception ez) {
AnalysisLogger.getLogger().debug("Dataspace->Status updater->Impossible to write computation information on the Workspace");
AnalysisLogger.getLogger().debug(ez);
}
}
};
public void updateComputationOnWS(float status, String exception, List<StoredData> inputData, List<File> generatedData) {
if (currentComputation != null) {
currentComputation.setStatus("" + status);
if (exception != null && exception.length() > 0)
currentComputation.setException(exception);
RunDataspaceManager rundm = new RunDataspaceManager(inputData,generatedData);
rundm.run();
/*
Thread t = new Thread(rundm);
t.start();
*/
}
}
@Execute
@ -533,6 +554,7 @@ public class AbstractEcologicalEngineMapper extends AbstractAnnotatedAlgorithm {
ComputationData computation = new ComputationData(config.getTaskID(), config.getAgent(), agent.getDescription(), agent.getInfrastructure().name(), startTime, endTime, "100", config.getTaskID(), config.getParam(ConfigurationManager.serviceUserNameParameterVariable), config.getGcubeScope(), this.getClass().getCanonicalName());
// post on WS
DataspaceManager manager = new DataspaceManager(config, computation, inputData, outputData, generatedFiles);
Thread t = new Thread(manager);
t.start();
AnalysisLogger.getLogger().debug("Provenance manager running");

11
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/ConfigurationManager.java
View File

@ -19,6 +19,7 @@ public class ConfigurationManager {
public static String webPersistencePathVariable = "";
public static String usernameParameter = "user.name";
public static String scopeParameter = "scope";
public static String tokenParameter = "usertoken";
public static String defaultScope= "/gcube/devsec";
public static String defaultUsername= "statistical.wps";
@ -57,6 +58,7 @@ public class ConfigurationManager {
private AlgorithmConfiguration config;
private String scope;
private String username;
private String token;
public String getScope() {
return scope;
@ -122,11 +124,20 @@ public class ConfigurationManager {
scope = "/" + scope;
username = (String) inputs.get(usernameParameter);
token = (String) inputs.get(tokenParameter);
AnalysisLogger.getLogger().debug("User name used by the client: " + username);
AnalysisLogger.getLogger().debug("User token used by the client: " + token);
if (username == null || username.trim().length() == 0)
throw new Exception("Error: user name parameter (user.name) not set! This violates e-Infrastructure security policies");
if (token == null || token.trim().length() == 0)
throw new Exception("Error: token parameter not set! This violates e-Infrastructure security policies");
config.setGcubeScope(scope);
config.setGcubeUserName(username);
config.setGcubeToken(token);
// DONE get username from request
config.setParam(serviceUserNameParameterVariable, username);
config.setParam(processingSessionVariable, "" + UUID.randomUUID());

26
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/TokenManager.java
View File

@ -1,15 +1,18 @@
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping;
import static org.gcube.common.authorization.client.Constants.authorizationService;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.AuthorizationProvider;
import org.gcube.common.authorization.library.provider.ClientInfo;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.provider.UserInfo;
import org.gcube.common.scope.api.ScopeProvider;
public class TokenManager {
String username;
String scope;
String token;
public String getScope(){
return scope;
@ -19,26 +22,35 @@ public class TokenManager {
return username;
}
public String getToken(){
return token;
}
public void getCredentials() {
try{
System.out.println("Retrieving token credentials");
scope = ScopeProvider.instance.get();
//scope = ScopeProvider.instance.get();
System.out.println("Credentials from the GHN: scope: "+scope);
//get username from SmartGears
ClientInfo token = (UserInfo) AuthorizationProvider.instance.get().getClient();
username = token.getId();
System.out.println("Credentials from the GHN: user: "+username);
ClientInfo tokenInfo = (UserInfo) AuthorizationProvider.instance.get().getClient();
username = tokenInfo.getId();
token = SecurityTokenProvider.instance.get();
AuthorizationEntry entry = authorizationService().get(token);
scope = entry.getContext();
System.out.println("Credentials from the GHN: user: "+username+" , "+scope+" , "+token);
}catch(Exception e){
e.printStackTrace();
System.out.println("Error Retrieving token credentials: "+e.getLocalizedMessage());
scope = null;
username= null;
}
if ((scope==null || username==null) && ConfigurationManager.isSimulationMode()){
scope = ConfigurationManager.defaultScope;
username = ConfigurationManager.defaultUsername;
}
System.out.println("Retrieved scope: "+scope+" Username: "+username +" SIMULATION MODE: "+ConfigurationManager.isSimulationMode());
System.out.println("Retrieved scope: "+scope+" Username: "+username +" Token "+token+" SIMULATION MODE: "+ConfigurationManager.isSimulationMode());
}

6
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/mapping/dataspace/DataspaceManager.java
View File

@ -176,7 +176,7 @@ public class DataspaceManager implements Runnable {
//fileItem.getProperties().addProperties(properties);
AnalysisLogger.getLogger().debug("Dataspace->WS OP file saved on the WS " + filenameonwsString);
url = fileItem.getPublicLink(true);
url = fileItem.getPublicLink(false);
AnalysisLogger.getLogger().debug("Dataspace->WS OP url produced for the file " + url);
data.payload = url;
@ -184,7 +184,7 @@ public class DataspaceManager implements Runnable {
in.close();
} catch (Exception e) {
AnalysisLogger.getLogger().debug("Dataspace->Error creating file " + e.getMessage());
AnalysisLogger.getLogger().debug(e);
//AnalysisLogger.getLogger().debug(e);
}
AnalysisLogger.getLogger().debug("Dataspace->File created " + filenameonwsString);
} else {
@ -194,7 +194,7 @@ public class DataspaceManager implements Runnable {
} catch (Throwable e) {
e.printStackTrace();
AnalysisLogger.getLogger().debug("Dataspace->Could not retrieve input payload " + data.payload+" - "+e.getLocalizedMessage());
AnalysisLogger.getLogger().debug(e);
//AnalysisLogger.getLogger().debug(e);
url = "payload was not made available for this dataset";
data.payload = url;
}

35
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/test/CreateTestTableFromFile.java Normal file
View File

@ -0,0 +1,35 @@
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.test;
import java.io.File;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure.DatabaseInfo;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.infrastructure.InfrastructureDialoguer;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping.InputsManager;
public class CreateTestTableFromFile {
public static void main(String[] args) throws Exception{
Regressor regressor = new Regressor();
AlgorithmConfiguration config = regressor.getConfig();
String tableName = "spread_test";
File tableFile = new File("C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\spread_input_dataset .csv");
String inputTableTemplate = TableTemplates.GENERIC.name();
String scope = "/gcube/devsec/devVRE";
System.out.println("Asking the infra for database in scope: "+scope);
InfrastructureDialoguer dialoguer = new InfrastructureDialoguer(scope);
DatabaseInfo supportDatabaseInfo = dialoguer.getDatabaseInfo("StatisticalManagerDataBase");
InputsManager manager = new InputsManager(null, config, "test");
manager.configSupportDatabaseParameters(supportDatabaseInfo);
System.out.println("Database retrieved: "+supportDatabaseInfo.url);
System.out.println("Creating: "+tableName);
manager.createTable(tableName, tableFile,manager.getConfig(), supportDatabaseInfo, inputTableTemplate);
System.out.println("DB coordinates: \nURL "+supportDatabaseInfo.url+"\n user "+supportDatabaseInfo.username+"\n password "+supportDatabaseInfo.password+"\n driver "+supportDatabaseInfo.driver+"\n dialect "+supportDatabaseInfo.dialect);
System.out.println("All done");
}
}

1
src/main/java/org/gcube/dataanalysis/wps/statisticalmanager/synchserver/web/ExecuteRequest.java
View File

@ -795,6 +795,7 @@ public class ExecuteRequest extends Request implements IObserver {
try {
is = executeDocument.newInputStream();
DatabaseFactory.getDatabase().insertRequest(wpsid, is, true);
//DatabaseFactory.getDatabase().insertRequest(wpsid, is, false);
} catch (Exception e) {
LOGGER.error("Exception storing ExecuteRequest", e);
} finally {