2016-04-01 13:09:40 +02:00
package org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mapping ;
import java.awt.Image ;
import java.awt.image.BufferedImage ;
import java.io.File ;
import java.io.FileWriter ;
import java.io.InputStream ;
import java.sql.Connection ;
import java.util.ArrayList ;
2017-05-05 10:08:44 +02:00
import java.util.Arrays ;
2016-04-01 13:09:40 +02:00
import java.util.LinkedHashMap ;
import java.util.List ;
import java.util.Map ;
import java.util.Properties ;
import java.util.UUID ;
import javax.imageio.ImageIO ;
import org.gcube.contentmanagement.graphtools.data.conversions.ImageTools ;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration ;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType ;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnTypesList ;
import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType ;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable ;
import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable ;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType ;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveTypesList ;
import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType ;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType ;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalTypeList ;
import org.gcube.dataanalysis.ecoengine.datatypes.TablesList ;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes ;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory ;
import org.gcube.dataanalysis.wps.statisticalmanager.synchserver.utils.FileManager ;
import org.postgresql.copy.CopyManager ;
import org.postgresql.core.BaseConnection ;
2017-05-05 10:08:44 +02:00
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
2016-04-01 13:09:40 +02:00
public class StatisticalTypeToWPSType {
2018-01-08 12:53:50 +01:00
2017-05-05 10:08:44 +02:00
private static Logger logger = LoggerFactory . getLogger ( StatisticalTypeToWPSType . class ) ;
2018-01-08 12:53:50 +01:00
2016-04-01 13:09:40 +02:00
public static Properties templates ;
static String ABSTRACT = " #ABSTRACT# " ;
static String TITLE = " #TITLE# " ;
static String CLASSNAME = " #CLASSNAME# " ;
static String ALLOWED = " #ALLOWED# " ;
static String DEFAULT = " #DEFAULT# " ;
static String ID = " #ID# " ;
static String IDMETHOD = " #IDMETHOD# " ;
2017-10-05 20:57:35 +02:00
static String METHOD_ORDER = " #ORDER_VALUE# " ;
2018-01-08 12:53:50 +01:00
2017-10-05 20:57:35 +02:00
private int orderValue = 0 ;
2018-01-08 12:53:50 +01:00
2016-04-01 13:09:40 +02:00
public LinkedHashMap < String , IOWPSInformation > inputSet = new LinkedHashMap < String , IOWPSInformation > ( ) ;
public LinkedHashMap < String , IOWPSInformation > outputSet = new LinkedHashMap < String , IOWPSInformation > ( ) ;
public List < File > generatedFiles = new ArrayList < File > ( ) ;
public List < String > generatedTables = new ArrayList < String > ( ) ;
public List < File > getGeneratedFiles ( ) {
return generatedFiles ;
}
public List < String > getGeneratedTables ( ) {
return generatedTables ;
}
public synchronized void getTemplates ( ) throws Exception {
if ( templates ! = null )
return ;
templates = new Properties ( ) ;
2018-01-08 12:53:50 +01:00
InputStream is = this . getClass ( ) . getClassLoader ( ) . getResourceAsStream ( " templates/classtemplate.properties " ) ;
templates . load ( is ) ;
is . close ( ) ;
2016-04-01 13:09:40 +02:00
}
public StatisticalTypeToWPSType ( ) throws Exception {
getTemplates ( ) ;
}
public String cleanID ( String name ) {
return name . replaceAll ( " [ \\ ] \\ [! \" #$%&'()*+, \\ ./:;<=>?@ \\ ^`{|}~-] " , " _ " ) ;
// return name;
}
public String convert2WPSType ( StatisticalType stype , boolean isinput , AlgorithmConfiguration config ) throws Exception {
if ( stype = = null )
return " " ;
String wpstype = null ;
String outputType = " " ;
TableTemplatesMapper mapper = new TableTemplatesMapper ( ) ;
String webpersistence = config . getParam ( ConfigurationManager . webPersistencePathVariable ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Using the foll. web persistence: " + webpersistence ) ;
2016-04-01 13:09:40 +02:00
String name = stype . getName ( ) ;
String classForname = stype . getClass ( ) . getSimpleName ( ) ;
if ( name = = null | | name . length ( ) = = 0 )
name = classForname . replace ( " . " , " " ) ;
String id = ( name ) ;
String abstractStr = stype . getDescription ( ) ! = null ? stype . getDescription ( ) : " " ;
String allowed = " " ;
String defaultVal = stype . getDefaultValue ( ) ! = null ? stype . getDefaultValue ( ) : " " ;
String content = null ;
String localcontent = null ;
String mimeType = " " ;
if ( stype instanceof PrimitiveType ) {
PrimitiveType ptype = ( PrimitiveType ) stype ;
PrimitiveTypes subtype = ptype . getType ( ) ;
switch ( subtype ) {
case STRING :
content = ( String ) ptype . getContent ( ) ;
outputType = " string " ;
mimeType = " text/plain " ;
break ;
case NUMBER : {
content = " " + ptype . getContent ( ) ;
String classname = ptype . getClassName ( ) ;
outputType = " integer " ;
mimeType = " text/plain " ;
if ( ! classname . equals ( Integer . class . getName ( ) ) ) {
outputType = " double " ;
}
break ;
}
case ENUMERATED : {
Object contentObj = ptype . getContent ( ) ;
content = Arrays . toString ( ( Object [ ] ) ptype . getContent ( ) ) ;
outputType = " enumerated " ;
mimeType = " text/plain " ;
Object [ ] allowedObjs = ( Object [ ] ) contentObj ;
allowed = " " ;
// generation of allowed values with check of the default value
String candidatedefaultvalue = " " ;
boolean defaultfound = false ;
for ( int i = 0 ; i < allowedObjs . length ; i + + ) {
String allowedS = ( " " + allowedObjs [ i ] ) . trim ( ) ;
allowed + = " \" " + allowedS + " \" " ;
if ( i = = 0 )
candidatedefaultvalue = allowedS ;
if ( allowedS . equals ( defaultVal ) )
defaultfound = true ;
if ( i < allowedObjs . length - 1 )
allowed + = " , " ;
}
if ( ! defaultfound )
defaultVal = candidatedefaultvalue ;
break ;
}
case FILE :
String filename = " " ;
if ( ptype . getContent ( ) ! = null ) {
String originalfile = ( ( File ) ptype . getContent ( ) ) . getAbsolutePath ( ) ;
( ( File ) ptype . getContent ( ) ) . getAbsolutePath ( ) ;
// search for the object in various locations
2017-05-05 10:08:44 +02:00
logger . debug ( " Searching for file in: " + originalfile ) ;
2016-04-01 13:09:40 +02:00
if ( ! new File ( originalfile ) . exists ( ) ) {
originalfile = new File ( config . getPersistencePath ( ) , ( ( File ) ptype . getContent ( ) ) . getName ( ) ) . getAbsolutePath ( ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Searching for file in persistence path: " + originalfile ) ;
2016-04-01 13:09:40 +02:00
if ( ! new File ( originalfile ) . exists ( ) ) {
originalfile = new File ( config . getConfigPath ( ) , ( ( File ) ptype . getContent ( ) ) . getName ( ) ) . getAbsolutePath ( ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Searching for file in config path: " + originalfile ) ;
2016-04-01 13:09:40 +02:00
}
}
if ( ! new File ( originalfile ) . exists ( ) ) {
2017-05-05 10:08:44 +02:00
logger . debug ( " The file does not exist! " + originalfile ) ;
2016-04-01 13:09:40 +02:00
} else {
2017-05-05 10:08:44 +02:00
logger . debug ( " The file exists! " + originalfile ) ;
2016-04-01 13:09:40 +02:00
filename = ( ( File ) ptype . getContent ( ) ) . getName ( ) ;
String filenameDest = System . currentTimeMillis ( ) + " _ " + filename ;
2018-01-08 12:53:50 +01:00
logger . debug ( " file destination for output is " + filenameDest ) ;
2016-04-01 13:09:40 +02:00
String destinationfile = new File ( webpersistence , filenameDest ) . getAbsolutePath ( ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Copying file into a temporary file: " + destinationfile ) ;
2018-01-08 12:53:50 +01:00
2017-05-05 10:08:44 +02:00
FileManager . fileCopy ( originalfile , destinationfile ) ;
2016-04-01 13:09:40 +02:00
content = config . getParam ( ConfigurationManager . webpathVariable ) + filenameDest ;
localcontent = destinationfile ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Web content associated to the file is: " + content ) ;
2016-04-01 13:09:40 +02:00
generatedFiles . add ( new File ( originalfile ) ) ;
}
}
if ( filename . toLowerCase ( ) . endsWith ( " .csv " ) | | filename . toLowerCase ( ) . endsWith ( " .txt " ) ) {
outputType = " csvFile " ;
mimeType = " text/csv " ;
} else {
outputType = " d4scienceFile " ;
mimeType = " application/d4science " ;
}
2017-05-05 10:08:44 +02:00
logger . debug ( " File managed correctly: Type: " + outputType + " mimetype: " + mimeType ) ;
2016-04-01 13:09:40 +02:00
break ;
case MAP : {
2017-05-05 10:08:44 +02:00
@SuppressWarnings ( " unchecked " )
2016-04-01 13:09:40 +02:00
Map < String , StatisticalType > subelements = ( Map < String , StatisticalType > ) ptype . getContent ( ) ;
wpstype = " " ;
int counter = 1 ;
for ( String subel : subelements . keySet ( ) ) {
StatisticalType stsub = subelements . get ( subel ) ;
String sclassForname = stsub . getClass ( ) . getSimpleName ( ) ;
if ( stsub . getName ( ) = = null | | stsub . getName ( ) . length ( ) = = 0 )
stsub . setName ( sclassForname . replace ( " . " , " " ) + counter ) ;
wpstype = wpstype + " \ n " + convert2WPSType ( stsub , isinput , config ) ;
counter + + ;
}
break ;
}
case BOOLEAN :
outputType = " boolean " ;
mimeType = " text/plain " ;
2016-06-14 12:25:56 +02:00
defaultVal = defaultVal . toLowerCase ( ) ;
2016-04-01 13:09:40 +02:00
break ;
case IMAGES : {
// content = ptype.getContent();
2017-05-05 10:08:44 +02:00
@SuppressWarnings ( " unchecked " )
2016-04-01 13:09:40 +02:00
Map < String , Image > subelements = ( Map < String , Image > ) ptype . getContent ( ) ;
wpstype = " " ;
if ( subelements ! = null & & subelements . size ( ) > 0 ) {
for ( String subel : subelements . keySet ( ) ) {
// Image stsub = subelements.get(subel);
outputType = " pngFile " ;
outputType + = isinput ? " Input " : " Output " ;
2017-10-05 20:57:35 +02:00
wpstype = wpstype + " \ n " + ( ( String ) templates . get ( outputType ) ) . replace ( ABSTRACT , subel ) . replace ( TITLE , subel ) . replace ( ID , subel ) . replace ( METHOD_ORDER , Integer . toString ( orderValue + + ) ) . replace ( IDMETHOD , cleanID ( id ) ) . replace ( DEFAULT , defaultVal ) ;
2016-04-01 13:09:40 +02:00
String imagefilename = new File ( webpersistence , subel + " _ " + UUID . randomUUID ( ) + " .png " ) . getAbsolutePath ( ) ;
BufferedImage bi = ImageTools . toBufferedImage ( subelements . get ( subel ) ) ;
File f = new File ( imagefilename ) ;
ImageIO . write ( bi , " png " , f ) ;
// upload on WS and get URL - TOO SLOW!
// String url =
// AbstractEcologicalEngineMapper.uploadOnWorkspaceAndGetURL(config.getGcubeScope(),
// imagefilename);
String url = config . getParam ( ConfigurationManager . webpathVariable ) + f . getName ( ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Got URL for the file " + url ) ;
2016-04-01 13:09:40 +02:00
IOWPSInformation info = new IOWPSInformation ( ) ;
info . setName ( subel ) ;
info . setAbstractStr ( subel ) ;
info . setAllowed ( allowed ) ;
info . setContent ( url ) ;
info . setLocalMachineContent ( imagefilename ) ;
info . setDefaultVal ( defaultVal ) ;
info . setMimetype ( " image/png " ) ;
info . setClassname ( classForname ) ;
generatedFiles . add ( f ) ;
if ( isinput )
inputSet . put ( subel , info ) ;
else
outputSet . put ( subel , info ) ;
}
}
break ;
}
default :
return null ;
}
} else if ( stype instanceof PrimitiveTypesList ) {
String format = ( ( PrimitiveTypesList ) stype ) . getClassName ( ) ;
format = format . substring ( format . lastIndexOf ( " . " ) + 1 ) ;
PrimitiveType pptype = new PrimitiveType ( ( ( PrimitiveTypesList ) stype ) . getClassName ( ) , null , ( ( PrimitiveTypesList ) stype ) . getType ( ) , name , abstractStr + " [a sequence of values separated by | ] (format: " + format + " ) " , defaultVal ) ;
pptype . setType ( PrimitiveTypes . STRING ) ;
pptype . setClassName ( String . class . getName ( ) ) ;
wpstype = convert2WPSType ( pptype , isinput , config ) ;
} else if ( stype instanceof ColumnType ) {
outputType = " string " ;
abstractStr + = " [the name of a column from " + ( ( ColumnType ) stype ) . getTableName ( ) + " ] " ;
mimeType = " text/plain " ;
} else if ( stype instanceof ColumnTypesList ) {
outputType = " string " ;
abstractStr + = " [a sequence of names of columns from " + ( ( ColumnTypesList ) stype ) . getTabelName ( ) + " separated by | ] " ;
mimeType = " text/plain " ;
} else if ( stype instanceof DatabaseType ) {
return null ;
} else if ( stype instanceof OutputTable ) {
// content = ((OutputTable) stype).getTableName();
String tablename = ( String ) ( ( OutputTable ) stype ) . getTableName ( ) ;
outputType = " csvFile " ;
mimeType = " text/csv " ;
String template = ( ( OutputTable ) stype ) . getTemplateNames ( ) . get ( 0 ) . name ( ) ;
abstractStr + = " [a http link to a table in UTF-8 ecoding following this template: " + mapper . linksMap . get ( template ) + " ] " ;
if ( tablename ! = null & & tablename . length ( ) > 0 ) {
generatedTables . add ( tablename ) ;
String localfile = new File ( webpersistence , tablename + UUID . randomUUID ( ) + " .csv " ) . getAbsolutePath ( ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Creating file " + localfile + " from table " + content ) ;
2016-04-01 13:09:40 +02:00
dumpTable ( localfile , tablename , " , " , config . databaseUserName , config . databasePassword , config . databaseURL ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " File " + localfile + " has been created " ) ;
2016-04-01 13:09:40 +02:00
// upload on WS and get URL
// String url =
// AbstractEcologicalEngineMapper.uploadOnWorkspaceAndGetURL(config.getGcubeScope(),
// localfile);
String url = config . getParam ( ConfigurationManager . webpathVariable ) + new File ( localfile ) . getName ( ) ;
2017-05-05 10:08:44 +02:00
logger . debug ( " Got URL for file " + url ) ;
2016-04-01 13:09:40 +02:00
content = url ;
localcontent = localfile ;
}
// upload on storage and get URL
} else if ( stype instanceof InputTable ) {
outputType = " csvFile " ;
mimeType = " text/csv " ;
String template = ( ( InputTable ) stype ) . getTemplateNames ( ) . get ( 0 ) . name ( ) ;
abstractStr + = " [a http link to a table in UTF-8 encoding following this template: " + mapper . linksMap . get ( template ) + " ] " ;
} else if ( stype instanceof ServiceType ) {
return null ;
} else if ( stype instanceof StatisticalTypeList ) {
return null ;
} else if ( stype instanceof TablesList ) {
outputType = " csvFile " ;
2018-01-08 12:53:50 +01:00
// outputType = "string";
2016-04-01 13:09:40 +02:00
String template = ( ( TablesList ) stype ) . getTemplates ( ) . get ( 0 ) . name ( ) ;
abstractStr + = " [a sequence of http links separated by | , each indicating a table in UTF-8 encoding following this template: " + mapper . linksMap . get ( template ) + " ] " ;
mimeType = " text/csv " ;
2018-01-08 12:53:50 +01:00
// mimeType = "text/plain";
2016-04-01 13:09:40 +02:00
}
outputType + = isinput ? " Input " : " Output " ;
if ( name = = null )
name = outputType ;
if ( wpstype = = null ) {
// wpstype = ((String) templates.get(outputType)).replace(ABSTRACT,
// abstractStr).replace(TITLE, name).replace(ID,
// id).replace(DEFAULT, defaultVal).replace(ALLOWED, allowed);
2017-10-05 20:57:35 +02:00
wpstype = ( ( String ) templates . get ( outputType ) ) . replace ( ABSTRACT , " Name of the parameter: " + name + " . " + abstractStr ) . replace ( TITLE , abstractStr ) . replace ( ID , id ) . replace ( METHOD_ORDER , Integer . toString ( orderValue + + ) ) . replace ( IDMETHOD , cleanID ( id ) ) . replace ( DEFAULT , defaultVal ) . replace ( ALLOWED , allowed ) ;
2016-04-01 13:09:40 +02:00
IOWPSInformation info = new IOWPSInformation ( ) ;
info . setName ( name ) ;
info . setAllowed ( allowed ) ;
info . setContent ( content ) ;
info . setDefaultVal ( defaultVal ) ;
info . setLocalMachineContent ( localcontent ) ;
info . setMimetype ( mimeType ) ;
info . setClassname ( classForname ) ;
info . setAbstractStr ( abstractStr ) ;
if ( localcontent ! = null )
generatedFiles . add ( new File ( localcontent ) ) ;
if ( isinput )
inputSet . put ( name , info ) ;
else
outputSet . put ( name , info ) ;
}
return wpstype . trim ( ) ;
}
public void dumpTable ( String filePath , String tablename , String delimiter , String username , String password , String databaseurl ) throws Exception {
Connection conn = DatabaseFactory . getDBConnection ( " org.postgresql.Driver " , username , password , databaseurl ) ;
CopyManager copyManager = new CopyManager ( ( BaseConnection ) conn ) ;
FileWriter fw = new FileWriter ( filePath ) ;
copyManager . copyOut ( String . format ( " COPY %s TO STDOUT WITH DELIMITER '%s' NULL AS '' CSV HEADER QUOTE AS ' \" ' " , tablename , delimiter ) , fw ) ;
conn . close ( ) ;
fw . close ( ) ;
}
2018-01-08 12:53:50 +01:00
2016-04-01 13:09:40 +02:00
/ *
* enum a { A , B } ;
* /
public static void main ( String [ ] args ) throws Exception {
// Object[] elements = a.values();
AlgorithmConfiguration config = new AlgorithmConfiguration ( ) ;
config . setGcubeScope ( " /gcube/devsec " ) ;
// config.setParam(AbstractEcologicalEngineMapper.processingSession,
// ""+UUID.randomUUID());
// config.setParam(AbstractEcologicalEngineMapper.serviceUserNameParameter,
// "wps.synch");
// AbstractEcologicalEngineMapper.uploadOnWorkspaceAndGetURL(config,new
// File( "./datasets/hcaf_d_mini.csv"), "test gp for WPS", "text/csv");
}
}