package org.gcube.portlets.user.databasesmanager.server; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; import java.net.URLStreamHandler; import javax.servlet.http.HttpSession; import org.apache.regexp.RE; import org.apache.regexp.RESyntaxException; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.contentmanager.storageclient.model.protocol.smp.SMPURLConnection; import org.gcube.data.analysis.statisticalmanager.proxies.StatisticalManagerDSL; import org.gcube.data.analysis.statisticalmanager.proxies.StatisticalManagerFactory; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMAlgorithm; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMComputationConfig; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMComputationRequest; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMGroupedAlgorithms; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMListGroupedAlgorithms; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMOperationStatus; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMParameter; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMParameters; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMResourceType; import org.gcube.data.analysis.statisticalmanager.stubs.types.SMTypeParameter; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMAbstractResource; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMComputation; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMEntries; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMFile; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMInputEntry; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMObject; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMOperationInfo; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMResource; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.SMTable; import org.gcube.data.analysis.statisticalmanager.stubs.types.schema.StatisticalServiceType; import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; import org.gcube.portlets.user.databasesmanager.client.GWTdbManagerService; import org.gcube.portlets.user.databasesmanager.client.datamodel.FileModel; import org.gcube.portlets.user.databasesmanager.client.datamodel.Parameter; import org.gcube.portlets.user.databasesmanager.client.datamodel.Result; import org.gcube.portlets.user.databasesmanager.client.datamodel.Row; import org.gcube.portlets.user.databasesmanager.server.util.SessionUtil; //import org.gcube.portlets.user.dbmanager.client.GWTdbManagerService; //import org.gcube.portlets.user.dbmanager.client.datamodel.FileModel; //import org.gcube.portlets.user.dbmanager.client.datamodel.Parameter; //import org.gcube.portlets.user.dbmanager.client.datamodel.Result; //import org.gcube.portlets.user.dbmanager.client.datamodel.Row; //import org.gcube.portlets.user.dbmanager.server.util.SessionUtil; import com.google.gwt.user.server.rpc.RemoteServiceServlet; import com.thoughtworks.xstream.XStream; public class GWTdbManagerServiceImpl extends RemoteServiceServlet implements GWTdbManagerService { private LinkedHashMap outputMap; private LinkedHashMap outputKey; private static Logger rootLogger = Logger .getLogger("GWTdbManagerServiceImpl"); public GWTdbManagerServiceImpl() { } public List getResource() throws Exception { List inputParameters = new ArrayList(); // List outputParameters = new ArrayList(); List outputParameters = new ArrayList(); // Parameter data = new Parameter(); // get list of algorithms List algorithms = new ArrayList(); algorithms = getDatabaseManagerAlgorithms(); // get algorithmId String algorithmId = null; for (int i = 0; i < algorithms.size(); i++) { if (algorithms.get(i).equals("LISTDBNAMES")) { algorithmId = algorithms.get(i); // System.out.println("algorithmId: " + algorithmId); rootLogger.log(Level.SEVERE, "algorithmId: " + algorithmId); } } // get input parameters of the algorithm inputParameters = getParameters(algorithmId); // System.out.println("getting input parameters"); rootLogger.log(Level.SEVERE, "getting input parameters"); for (int i = 0; i < inputParameters.size(); i++) { // System.out.println(inputParameters.get(i).getName()); rootLogger.log(Level.INFO, inputParameters.get(i).getName()); } String computationId = startComputation(algorithmId, inputParameters); // retrieve data // System.out.println("output data retrieved"); rootLogger.log(Level.SEVERE, "output data retrieved"); for (int i = 0; i < outputMap.size(); i++) { // outputParameters.add(outputMap.get(String.valueOf(i))); FileModel obj = new FileModel(outputMap.get(String.valueOf(i))); // obj.setIsLoaded(true); outputParameters.add(obj); } return outputParameters; } @Override public LinkedHashMap getDBInfo(String resourceName) throws Exception { List inputParameters = new ArrayList(); // List outputParameters = new ArrayList(); // List outputParameters = new ArrayList(); LinkedHashMap outputParameters = new LinkedHashMap(); // get list of algorithms List algorithms = new ArrayList(); algorithms = getDatabaseManagerAlgorithms(); // get algorithmId String algorithmId = null; for (int i = 0; i < algorithms.size(); i++) { if (algorithms.get(i).equals("LISTDBINFO")) { algorithmId = algorithms.get(i); // System.out.println("algorithmId: " + algorithmId); rootLogger.log(Level.SEVERE, "algorithmId: " + algorithmId); } } // get input parameters of the algorithm rootLogger.log(Level.SEVERE, "getting input parameters"); inputParameters = getParameters(algorithmId); for (int i = 0; i < inputParameters.size(); i++) { // System.out.println(inputParameters.get(i).getName()); inputParameters.get(i).setValue(resourceName); rootLogger.log(Level.INFO, inputParameters.get(i).getName()); } String computationId = startComputation(algorithmId, inputParameters); // retrieve data // System.out.println("output data retrieved"); rootLogger.log(Level.SEVERE, "output data retrieved"); for (int i = 0; i < outputMap.size(); i++) { // outputParameters.add(outputKey.get(String.valueOf(i)) + " " // + outputMap.get(String.valueOf(i))); FileModel obj = new FileModel(outputMap.get(String.valueOf(i))); // obj.setIsLoaded(true); // outputParameters.add(obj); // System.out.println("value: " + outputMap.get(String.valueOf(i))); // System.out.println("key: " + outputKey.get(String.valueOf(i))); outputParameters.put(outputKey.get(String.valueOf(i)), obj); } return outputParameters; } @Override public List getDBSchema(LinkedHashMap dataInput) throws Exception { List inputParameters = new ArrayList(); // List outputParameters = new ArrayList(); List outputParameters = new ArrayList(); // get list of algorithms List algorithms = new ArrayList(); algorithms = getDatabaseManagerAlgorithms(); // get algorithmId String algorithmId = null; for (int i = 0; i < algorithms.size(); i++) { if (algorithms.get(i).equals("LISTDBSCHEMA")) { algorithmId = algorithms.get(i); // System.out.println("algorithmId: " + algorithmId); rootLogger.log(Level.SEVERE, "algorithmId: " + algorithmId); } } // get input parameters of the algorithm rootLogger.log(Level.SEVERE, "getting input parameters"); inputParameters = getParameters(algorithmId); for (int i = 0; i < inputParameters.size(); i++) { // System.out.println(inputParameters.get(i).getName()); rootLogger.log(Level.INFO, inputParameters.get(i).getName()); } inputParameters.get(0).setValue(dataInput.get("ResourceName")); inputParameters.get(1).setValue(dataInput.get("DatabaseName")); String computationId = startComputation(algorithmId, inputParameters); // retrieve data // System.out.println("output data retrieved"); rootLogger.log(Level.SEVERE, "output data retrieved"); for (int i = 0; i < outputMap.size(); i++) { // outputParameters.add(outputKey.get(String.valueOf(i)) + " " // + outputMap.get(String.valueOf(i))); FileModel obj = new FileModel(outputMap.get(String.valueOf(i))); obj.setIsSchema(true); // obj.setIsLoaded(true); outputParameters.add(obj); } return outputParameters; } @Override public List getTables(LinkedHashMap dataInput) throws Exception { List inputParameters = new ArrayList(); // List outputParameters = new ArrayList(); List outputParameters = new ArrayList(); // get list of algorithms List algorithms = new ArrayList(); algorithms = getDatabaseManagerAlgorithms(); // get algorithmId String algorithmId = null; for (int i = 0; i < algorithms.size(); i++) { if (algorithms.get(i).equals("LISTTABLES")) { algorithmId = algorithms.get(i); // System.out.println("algorithmId: " + algorithmId); rootLogger.log(Level.SEVERE, "algorithmId: " + algorithmId); } } // get input parameters of the algorithm rootLogger.log(Level.SEVERE, "getting input parameters"); inputParameters = getParameters(algorithmId); for (int i = 0; i < inputParameters.size(); i++) { // System.out.println(inputParameters.get(i).getName()); rootLogger.log(Level.INFO, inputParameters.get(i).getName()); } inputParameters.get(0).setValue(dataInput.get("ResourceName")); inputParameters.get(1).setValue(dataInput.get("DatabaseName")); inputParameters.get(2).setValue(dataInput.get("SchemaName")); // System.out.println("size outputMap pre computation: " // + outputMap.size()); String computationId = startComputation(algorithmId, inputParameters); // retrieve data // System.out.println("output data retrieved"); rootLogger.log(Level.SEVERE, "output data retrieved"); System.out.println("size outputMap: " + outputMap.size()); for (int i = 0; i < outputMap.size(); i++) { // outputParameters.add(outputKey.get(String.valueOf(i)) + " " // + outputMap.get(String.valueOf(i))); FileModel obj = new FileModel(outputMap.get(String.valueOf(i))); // obj.setIsLoaded(true); // obj.setIsTable(true); outputParameters.add(obj); } return outputParameters; } @Override public List submitQuery(LinkedHashMap dataDB, String query, boolean valueReadOnlyQuery, boolean smartCorrectionQuery, String language) throws Exception { // System.out.println("language detected: " + language); rootLogger.log(Level.SEVERE, "language used for query conversion: " + language); List inputParameters = new ArrayList(); // LinkedHashMap output = new LinkedHashMap(); List output = new ArrayList(); // get list of algorithms List algorithms = new ArrayList(); algorithms = getDatabaseManagerAlgorithms(); // get algorithmId String algorithmId = null; for (int i = 0; i < algorithms.size(); i++) { if (algorithms.get(i).equals("SUBMITQUERY")) { algorithmId = algorithms.get(i); // System.out.println("algorithmId: " + algorithmId); rootLogger.log(Level.SEVERE, "algorithmId: " + algorithmId); } } // get input parameters of the algorithm rootLogger.log(Level.SEVERE, "getting input parameters"); inputParameters = getParameters(algorithmId); // for (int i = 0; i < inputParameters.size(); i++) { // // System.out.println(inputParameters.get(i).getName()); // System.out.println(inputParameters.get(i).getDefaultValue()); // // } for (int i = 0; i < inputParameters.size(); i++) { // System.out.println(inputParameters.get(i).getName()); // System.out.println(inputParameters.get(i).getDefaultValue()); rootLogger.log(Level.INFO, inputParameters.get(i).getName()); } inputParameters.get(0).setValue(dataDB.get("ResourceName")); inputParameters.get(1).setValue(dataDB.get("DatabaseName")); inputParameters.get(2).setValue(String.valueOf(valueReadOnlyQuery)); inputParameters.get(3).setValue(String.valueOf(smartCorrectionQuery)); inputParameters.get(4).setValue(language); inputParameters.get(5).setValue(query); // System.out.println("size outputMap pre computation: " // + outputMap.size()); String computationId = startComputation(algorithmId, inputParameters); // retrieve data // System.out.println("output data retrieved"); // System.out.println("size outputMap: " + outputMap.size()); rootLogger.log(Level.SEVERE, "output data retrieved"); rootLogger.log(Level.SEVERE, "output data size: " + outputMap.size()); for (int i = 0; i < outputMap.size(); i++) { Result row = new Result(outputKey.get(String.valueOf(i)), outputMap.get(String.valueOf(i))); output.add(row); // outputParameters.add(outputKey.get(String.valueOf(i)) + " " // + outputMap.get(String.valueOf(i))); // FileModel obj = new FileModel(outputMap.get(String.valueOf(i))); // outputParameters.add(obj); } // for (int i = 0; i < outputMap.size(); i++) { // // // outputParameters.add(outputKey.get(String.valueOf(i)) + " " // // + outputMap.get(String.valueOf(i))); // // FileModel obj = new FileModel(outputMap.get(String.valueOf(i))); // outputParameters.add(obj); // // } return output; } public List parseCVSString(List result, List attrNames) throws Exception { List rows = new ArrayList(); if (result != null) { for (int i = 0; i < result.size(); i++) { List attrValues = parse(result.get(i).getValue()); Row element = new Row(attrNames, attrValues); rows.add(element); } } return rows; // code worked // List values = parse(row); // // for (int i = 0; i < values.size(); i++) { // // System.out.println("vaalue: " + values.get(i)); // // } // // Row element = new Row(attributes, values); // // return element; // List rowElements = new ArrayList(); // // for (int i=0;i attributes = parse(result.get(i).getValue()); // // Row element=new Row(attributes); // // rowElements.add(element); // } // // System.out.println("server rowElement size: " + rowElements.size()); // // return rowElements; } private List parse(String row) throws Exception { String delimiter = ","; // System.out.println("row: " + row); List elements = new ArrayList(); String phrase = row; int idxdelim = -1; boolean quot = false; phrase = phrase.trim(); while ((idxdelim = phrase.indexOf(delimiter)) >= 0) { quot = phrase.startsWith("\""); if (quot) { phrase = phrase.substring(1); String quoted = ""; if (phrase.startsWith("\"")) phrase = phrase.substring(1); else { RE regexp = new RE("[^\\\\]\""); boolean matching = regexp.match(phrase); if (matching) { int i0 = regexp.getParenStart(0); quoted = phrase.substring(0, i0 + 1).trim(); phrase = phrase.substring(i0 + 2).trim(); } } if (phrase.startsWith(delimiter)) phrase = phrase.substring(1); elements.add(quoted); } else { elements.add(phrase.substring(0, idxdelim)); phrase = phrase.substring(idxdelim + 1).trim(); } // System.out.println("server token: " + phrase); } if (phrase.startsWith("\"")) phrase = phrase.substring(1); if (phrase.endsWith("\"")) phrase = phrase.substring(0, phrase.length() - 1); elements.add(phrase); // System.out.println("size: " + elements.size()); return elements; } private List getDatabaseManagerAlgorithms() throws Exception { ArrayList dbAlg = new ArrayList(); StatisticalManagerFactory factory = getFactory(); // get and print algorithms SMListGroupedAlgorithms groups = factory.getAlgorithms(); if (groups == null) // System.out.println("GROUPS OF ALGORITHMS IS NULL!"); rootLogger.log(Level.SEVERE,"GROUPS OF ALGORITHMS IS NULL!"); else // System.out.println("GROUPS OF ALGORITHMS IS NOT NULL!"); rootLogger.log(Level.SEVERE, "GROUPS OF ALGORITHMS IS NOT NULL!"); // get list categories for (SMGroupedAlgorithms group : groups.thelist()) { // if (group.category().equals("DATABASES")) { for (SMAlgorithm algorithm : group.thelist()) { // if (algorithm.name().equals("LISTDBINFO")) { // System.out.println(algorithm.name()); dbAlg.add(algorithm.name()); // } } } return dbAlg; } private List getParameters(String algorithmId) { // System.out.println("algorithmId: " + algorithmId); StatisticalManagerFactory factory = getFactory(); SMParameters smParams = factory.getAlgorithmParameters(algorithmId); List params = new ArrayList(); for (SMParameter smParam : smParams.list()) { SMTypeParameter smType = smParam.type(); StatisticalServiceType smTypeName = smType.name(); String paramName = smParam.name(); String paramDescription = smParam.description(); String defaultValue = smParam.defaultValue(); String objectType = smType.values().get(0); // System.out.println("GWTdbManagerServiceImpl->parameters"); //CHECK // rootLogger.log(Level.INFO, "parameters: "); // rootLogger.log(Level.INFO, paramName); // rootLogger.log(Level.INFO, paramDescription); // rootLogger.log(Level.INFO, objectType); // rootLogger.log(Level.INFO, defaultValue); // System.out.println(paramName); // System.out.println(paramDescription); // System.out.println(objectType); // System.out.println(defaultValue); Parameter objectParam = new Parameter(paramName, paramDescription, objectType, defaultValue); params.add(objectParam); } return params; } private String startComputation(String algorithmName, List parameters) throws Exception { SMComputationConfig config = new SMComputationConfig(); SMInputEntry[] list = new SMInputEntry[parameters.size()]; int i = 0; for (Parameter p : parameters) list[i++] = new SMInputEntry(p.getName(), p.getValue()); config.parameters(new SMEntries(list)); config.algorithm(algorithmName); // create a computation request SMComputationRequest request = new SMComputationRequest(); request.user(getUsername()); request.config(config); try { StatisticalManagerFactory factory = getFactory(); String computationId = factory.executeComputation(request); float percentage = 0; String scope = getScope(); String username = getUsername(); while (percentage < 100) { percentage = checkComputationStatus(scope, computationId, username); Thread.sleep(1000); } return computationId; } catch (Exception e) { // e.printStackTrace(); throw e; } } private float checkComputationStatus(String scope, String computationId, String user) throws Exception { ScopeProvider.instance.set(scope); StatisticalManagerFactory factory = StatisticalManagerDSL .createStateful().build(); SMComputation computation = factory.getComputation(computationId); SMOperationStatus status = SMOperationStatus.values()[computation .operationStatus()]; // this.outputMap=null; float percentage = 0; if (status == SMOperationStatus.RUNNING) { // System.out.println("GWTdbManagerServiceImpl->RUNNING"); rootLogger.log(Level.SEVERE, "RUNNING"); SMOperationInfo infos = factory.getComputationInfo(computationId, user); // percentage = Float.parseFloat(infos.percentage()); // System.out.println("GWTdbManagerServiceImpl->Percentage:" + // percentage); // computation = factory.getComputation(computationId); status = SMOperationStatus.values()[computation.operationStatus()]; } else if ((status == SMOperationStatus.COMPLETED) || (status == SMOperationStatus.FAILED)) { // System.out.println("GWTdbManagerServiceImpl->COMPLETED OR FAILED"); rootLogger.log(Level.SEVERE, "COMPLETED OR FAILED"); SMAbstractResource abstractResource = computation .abstractResource(); SMResource smResource = abstractResource.resource(); int resourceTypeIndex = smResource.resourceType(); SMResourceType smResType = SMResourceType.values()[resourceTypeIndex]; //TODO: ECXEPTION MANAGEMENT // if (SMResourceType.ERROR){ // // // } //throws(smResource.description()); displayOutput(smResource, smResType); // System.out.println("GWTdbManagerServiceImpl-> SM resource Name: " // + smResource.name()); rootLogger.log(Level.SEVERE, "SM resource Name: " + smResource.name()); // System.out.println("GWTdbManagerServiceImpl-> SM resource ID: " // + smResource.resourceId()); rootLogger.log(Level.SEVERE, "SM resource ID: " + smResource.resourceId()); // System.out // .println("GWTdbManagerServiceImpl-> SM resource Description: " // + smResource.description()); rootLogger.log(Level.SEVERE, "SM resource Description: " + smResource.description()); percentage = 100; } return percentage; } private void displayOutput(SMResource smResource, SMResourceType smResType) throws Exception { // System.out.println("displayOutput"); rootLogger.log(Level.SEVERE, "displayOutput"); outputMap = new LinkedHashMap(); if (smResType.equals(SMResourceType.OBJECT)) { // switch (smResType) { // case FILE: // SMFile fileRes = (SMFile) smResource; // System.out.println("Output is a file"); // break; // case OBJECT: SMObject objRes = (SMObject) smResource; if (objRes.name().contentEquals(PrimitiveTypes.MAP.toString())) { // System.out.println("Output is a map"); rootLogger.log(Level.SEVERE, "Output is a map"); // Map outputMap = getMap(objRes); this.outputMap = (LinkedHashMap) getMap(objRes); } else if (objRes.name().contentEquals( PrimitiveTypes.IMAGES.toString())) { // System.out.println("Output are images"); rootLogger.log(Level.SEVERE, "Output are images"); } else // System.out.println("Output is other"); rootLogger.log(Level.SEVERE, "Output is other"); } //EXCEPTION MANAGEMENT if(smResType.equals(SMResourceType.ERROR)) { throw new Exception(smResource.description()); } // break; // case TABULAR: // SMTable tableRes = (SMTable) smResource; // System.out.println("Output is a table"); // break; // } } private Map getMap(SMObject objRes) throws Exception { // System.out.println("getMap"); rootLogger.log(Level.SEVERE, "getMap"); outputKey = new LinkedHashMap(); InputStream is = getStorageClientInputStream(objRes.url()); // object serializer XStream xstream = new XStream(); xstream.alias( "org.gcube_system.namespaces.data.analysis.statisticalmanager.types.SMObject", SMObject.class); xstream.alias( "org.gcube_system.namespaces.data.analysis.statisticalmanager.types.SMFile", SMFile.class); xstream.alias( "org.gcube_system.namespaces.data.analysis.statisticalmanager.types.SMResource", SMResource.class); xstream.alias( "org.gcube_system.namespaces.data.analysis.statisticalmanager.types.SMTable", SMTable.class); @SuppressWarnings("unchecked") Map smMap = (Map) (xstream .fromXML(is)); is.close(); LinkedHashMap outputmap = new LinkedHashMap(); int i = 0; for (String key : smMap.keySet()) { // add key value outputKey.put(String.valueOf(i), key); SMResource smres = smMap.get(key); int resourceTypeIndex = smres.resourceType(); SMResourceType smsubResType = SMResourceType.values()[resourceTypeIndex]; // System.out.println("GWTdbManagerServiceImpl-> ResourceType:" // + smsubResType); rootLogger.log(Level.INFO, "ResourceType: " + smsubResType); if (smsubResType == SMResourceType.OBJECT) { SMObject obje = (SMObject) smres; String outstring = obje.url(); // System.out.println("key :" + key); rootLogger.log(Level.INFO, "key: " + smsubResType); // System.out.println("Object:" // + new String(outstring.getBytes(), "UTF-8")); rootLogger.log(Level.INFO, "Object: " + new String(outstring.getBytes(), "UTF-8")); // outputmap.put(key, outstring); outputmap.put(String.valueOf(i), outstring); i++; } } // System.out.println("Map retrieved with size: " + smMap.size()); rootLogger.log(Level.SEVERE, "Map retrieved with size: " + smMap.size()); return outputmap; } private InputStream getStorageClientInputStream(String url) throws Exception { URL u = new URL(null, url, new URLStreamHandler() { @Override protected URLConnection openConnection(URL u) throws IOException { return new SMPURLConnection(u); } }); return u.openConnection().getInputStream(); } private StatisticalManagerFactory getFactory() { HttpSession httpSession = this.getThreadLocalRequest().getSession(); return SessionUtil.getFactory(httpSession); } private String getUsername() { HttpSession httpSession = this.getThreadLocalRequest().getSession(); return SessionUtil.getUsername(httpSession); } private String getScope() { HttpSession httpSession = this.getThreadLocalRequest().getSession(); return SessionUtil.getScope(httpSession); } }