gCubeSystem
/
ecological-engine
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@71234 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2013-03-12 13:48:46 +00:00
parent 3ef30fe289
commit f5217598a8
43 changed files with 699 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cfgLexical/hibernate.cfg.xml Normal file
View File

@ -0,0 +1,18 @@
<?xml version='1.0' encoding='UTF-8'?>
<hibernate-configuration>
<session-factory>
<property name="connection.driver_class">org.postgresql.Driver</property>
<property name="connection.provider_class">org.hibernate.connection.C3P0ConnectionProvider</property>
<property name="connection.url">jdbc:postgresql://localhost/testdb</property>
<property name="connection.username">gcube</property>
<property name="connection.password">d4science2</property>
<!-- <property name="dialect">org.hibernatespatial.postgis.PostgisDialect</property>-->
<property name="dialect">org.hibernate.dialect.PostgreSQLDialect</property>
<property name="transaction.factory_class">org.hibernate.transaction.JDBCTransactionFactory</property>
<property name="c3p0.timeout">0</property>
<property name="c3p0.max_size">1</property>
<property name="c3p0.max_statements">0</property>
<property name="c3p0.min_size">1</property>
<property name="current_session_context_class">thread</property>
</session-factory>
</hibernate-configuration>

20
cfgLexical/lexicalGuesser.properties Normal file
View File

@ -0,0 +1,20 @@
#Percentage threshold for discarding a category
categoryDiscardThreshold=0
#Percentage threshold for accepting similarity between a single Time series entry and a reference entry
entryAcceptanceThreshold=50
#Size of a comparison chunk
chunkSize=50
#Number of chunks to take from Time series for performing comparison respect to reference data; if set to -1 all chunks will be analyzed
timeSeriesChunksToTake=1
#Number of chunks to take from Reference for performing comparison Time Series Elements; if set to -1 all chunks will be analyzed
referenceChunksToTake=5
#Use random choice for chunks selection = true |false
randomTake=true
#Use Simple String Match as distance calculation
useSimpleDistance=false
#Number Of Threads to use
numberOfThreadsToUse=5
#if two final scores differ for more than this percentage, prune the lower result
categoryDiscardDifferencialThreshold = 50
#maximum difference between a result and the best result
singleEntryRecognitionMaxDeviation = 40

8
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/core/Engine.java
View File

@ -21,7 +21,7 @@ import org.hibernate.SessionFactory;
public class Engine { public class Engine {
private String ConfigurationFileNameLocal = "hibernate.cfg.xml"; private String ConfigurationFileNameLocal = "lexicalguesser/hibernate.cfg.xml";
private SessionFactory referenceDBSession; private SessionFactory referenceDBSession;
public ArrayList<String> bestCategories; public ArrayList<String> bestCategories;
@ -58,24 +58,22 @@ public class Engine {
return referenceDBSession; return referenceDBSession;
} }
public void resetEngine(LexicalEngineConfiguration Config,String ColumnFilter,String configPath){ public void resetEngine(LexicalEngineConfiguration Config,String ColumnFilter){
config = Config; config = Config;
scoresTable = new HashMap<String, CategoryScores>(); scoresTable = new HashMap<String, CategoryScores>();
bestCategories = new ArrayList<String>(); bestCategories = new ArrayList<String>();
bestColumns = new ArrayList<String>(); bestColumns = new ArrayList<String>();
bestScores = new ArrayList<Double>(); bestScores = new ArrayList<Double>();
columnFilter = ColumnFilter; columnFilter = ColumnFilter;
// ConfigurationFileNameLocal = configPath+"/"+ConfigurationFileNameLocal;
} }
public Engine(LexicalEngineConfiguration Config,String ColumnFilter,String configPath) { public Engine(LexicalEngineConfiguration Config,String ColumnFilter) {
config = Config; config = Config;
scoresTable = new HashMap<String, CategoryScores>(); scoresTable = new HashMap<String, CategoryScores>();
bestCategories = new ArrayList<String>(); bestCategories = new ArrayList<String>();
bestColumns = new ArrayList<String>(); bestColumns = new ArrayList<String>();
bestScores = new ArrayList<Double>(); bestScores = new ArrayList<Double>();
columnFilter = ColumnFilter; columnFilter = ColumnFilter;
ConfigurationFileNameLocal = configPath+"/"+ConfigurationFileNameLocal;
} }
public void calcLike(CategoryOrderedList col, String unknownSeriesName, String unknownSeriesColumn) { public void calcLike(CategoryOrderedList col, String unknownSeriesName, String unknownSeriesColumn) {

165
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/core/LexicalEngineConfiguration.java
View File

@ -1,8 +1,11 @@
package org.gcube.contentmanagement.lexicalmatcher.analysis.core; package org.gcube.contentmanagement.lexicalmatcher.analysis.core;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Properties; import java.util.Properties;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category;
public class LexicalEngineConfiguration { public class LexicalEngineConfiguration {
@ -10,6 +13,19 @@ public class LexicalEngineConfiguration {
Properties props = new Properties(); Properties props = new Properties();
FileInputStream fis = new FileInputStream(absoluteFilePath); FileInputStream fis = new FileInputStream(absoluteFilePath);
props.load(fis); props.load(fis);
setProperties(props);
fis.close();
}
public void configureByStream(String file) throws Exception {
Properties props = new Properties();
InputStream is = ClassLoader.getSystemResourceAsStream(file);
props.load(is);
setProperties(props);
is.close();
}
private void setProperties(Properties props) {
categoryDiscardThreshold = Float.parseFloat(props.getProperty("categoryDiscardThreshold")); categoryDiscardThreshold = Float.parseFloat(props.getProperty("categoryDiscardThreshold"));
entryAcceptanceThreshold = Integer.parseInt(props.getProperty("entryAcceptanceThreshold")); entryAcceptanceThreshold = Integer.parseInt(props.getProperty("entryAcceptanceThreshold"));
chunkSize = Integer.parseInt(props.getProperty("chunkSize")); chunkSize = Integer.parseInt(props.getProperty("chunkSize"));
@ -20,11 +36,8 @@ public class LexicalEngineConfiguration {
numberOfThreadsToUse = Integer.parseInt(props.getProperty("numberOfThreadsToUse")); numberOfThreadsToUse = Integer.parseInt(props.getProperty("numberOfThreadsToUse"));
categoryDiscardDifferencialThreshold = Float.parseFloat(props.getProperty("categoryDiscardDifferencialThreshold")); categoryDiscardDifferencialThreshold = Float.parseFloat(props.getProperty("categoryDiscardDifferencialThreshold"));
singleEntryRecognitionMaxDeviation = Float.parseFloat(props.getProperty("singleEntryRecognitionMaxDeviation")); singleEntryRecognitionMaxDeviation = Float.parseFloat(props.getProperty("singleEntryRecognitionMaxDeviation"));
fis.close();
} }
public void setCategoryDiscardThreshold(float categoryDiscardThreshold) { public void setCategoryDiscardThreshold(float categoryDiscardThreshold) {
this.categoryDiscardThreshold = categoryDiscardThreshold; this.categoryDiscardThreshold = categoryDiscardThreshold;
} }
@ -41,8 +54,6 @@ public class LexicalEngineConfiguration {
return entryAcceptanceThreshold; return entryAcceptanceThreshold;
} }
public void setCategoryDiscardDifferencialThreshold(float categoryDiscardDifferencialThreshold) { public void setCategoryDiscardDifferencialThreshold(float categoryDiscardDifferencialThreshold) {
this.categoryDiscardDifferencialThreshold = categoryDiscardDifferencialThreshold; this.categoryDiscardDifferencialThreshold = categoryDiscardDifferencialThreshold;
} }
@ -91,7 +102,6 @@ public class LexicalEngineConfiguration {
return useSimpleDistance; return useSimpleDistance;
} }
public void setNumberOfThreadsToUse(int numberOfThreadsToUse) { public void setNumberOfThreadsToUse(int numberOfThreadsToUse) {
this.numberOfThreadsToUse = numberOfThreadsToUse; this.numberOfThreadsToUse = numberOfThreadsToUse;
} }
@ -120,7 +130,7 @@ public class LexicalEngineConfiguration {
public Boolean useSimpleDistance = null; public Boolean useSimpleDistance = null;
public int numberOfThreadsToUse = -Integer.MIN_VALUE; public int numberOfThreadsToUse = -Integer.MIN_VALUE;
//database parameters // database parameters
public String databaseDriver = null; public String databaseDriver = null;
public String databaseURL = null; public String databaseURL = null;
public String databaseUserName = null; public String databaseUserName = null;
@ -129,142 +139,123 @@ public class LexicalEngineConfiguration {
public String databaseIdleConnectionTestPeriod = null; public String databaseIdleConnectionTestPeriod = null;
public String databaseAutomaticTestTable = null; public String databaseAutomaticTestTable = null;
//reference data parameters // reference data parameters
public String referenceTable = null; public String referenceTable = null;
public String referenceColumn = null; public String referenceColumn = null;
public String idColumn= null; public String idColumn = null;
public String nameHuman = null; public String nameHuman = null;
public String description = null; public String description = null;
public ArrayList<Category> categories = null;
public void mergeConfig(LexicalEngineConfiguration config){ public void setCategories(ArrayList<Category> categories) {
this.categories = categories;
if (config.getCategoryDiscardDifferencialThreshold()!=-Float.MIN_VALUE)
setCategoryDiscardDifferencialThreshold(config.getCategoryDiscardDifferencialThreshold());
if (config.getSingleEntryRecognitionMaxDeviation()!=-Float.MIN_VALUE)
setSingleEntryRecognitionMaxDeviation(config.getSingleEntryRecognitionMaxDeviation());
if (config.getCategoryDiscardThreshold()!=-Float.MIN_VALUE)
setCategoryDiscardThreshold(config.getCategoryDiscardThreshold());
if (config.getChunkSize()!=-Integer.MIN_VALUE)
setChunkSize(config.getChunkSize());
if (config.getEntryAcceptanceThreshold()!=-Float.MIN_VALUE)
setEntryAcceptanceThreshold(config.getEntryAcceptanceThreshold());
if (config.getNumberOfThreadsToUse()!=-Integer.MIN_VALUE)
setNumberOfThreadsToUse(config.getNumberOfThreadsToUse());
if (config.getReferenceChunksToTake()!=-Integer.MIN_VALUE)
setReferenceChunksToTake(config.getReferenceChunksToTake());
if (config.getTimeSeriesChunksToTake()!=-Integer.MIN_VALUE)
setTimeSeriesChunksToTake(config.getTimeSeriesChunksToTake());
if (config.randomTake!= null)
setRandomTake(config.isRandomTake());
if (config.useSimpleDistance!=null)
setUseSimpleDistance(config.isUseSimpleDistance());
//database information merge
if (config.databaseDriver!=null)
setDatabaseDriver(config.databaseDriver);
if (config.databaseDialect!=null)
setDatabaseDialect(config.databaseDialect);
if (config.databaseAutomaticTestTable!=null)
setDatabaseAutomaticTestTable(config.databaseAutomaticTestTable);
if (config.databaseIdleConnectionTestPeriod!=null)
setDatabaseIdleConnectionTestPeriod(config.databaseIdleConnectionTestPeriod);
if (config.databaseUserName!=null)
setDatabaseUserName(config.databaseUserName);
if (config.databasePassword!=null)
setDatabasePassword(config.databasePassword);
if (config.databaseURL!=null)
setDatabaseURL(config.databaseURL);
if (config.referenceTable!=null)
setReferenceTable(config.referenceTable);
if (config.referenceColumn!=null)
setReferenceColumn(config.referenceColumn);
if (config.idColumn!=null)
setIdColumn(config.idColumn);
if (config.nameHuman!=null)
setNameHuman(config.nameHuman);
if (config.description!=null)
setDescription(config.description);
} }
public ArrayList<Category> getCategories() {
return this.categories;
}
public void mergeConfig(LexicalEngineConfiguration config) {
if (config.getCategoryDiscardDifferencialThreshold() != -Float.MIN_VALUE)
setCategoryDiscardDifferencialThreshold(config.getCategoryDiscardDifferencialThreshold());
if (config.getSingleEntryRecognitionMaxDeviation() != -Float.MIN_VALUE)
setSingleEntryRecognitionMaxDeviation(config.getSingleEntryRecognitionMaxDeviation());
if (config.getCategoryDiscardThreshold() != -Float.MIN_VALUE)
setCategoryDiscardThreshold(config.getCategoryDiscardThreshold());
if (config.getChunkSize() != -Integer.MIN_VALUE)
setChunkSize(config.getChunkSize());
if (config.getEntryAcceptanceThreshold() != -Float.MIN_VALUE)
setEntryAcceptanceThreshold(config.getEntryAcceptanceThreshold());
if (config.getNumberOfThreadsToUse() != -Integer.MIN_VALUE)
setNumberOfThreadsToUse(config.getNumberOfThreadsToUse());
if (config.getReferenceChunksToTake() != -Integer.MIN_VALUE)
setReferenceChunksToTake(config.getReferenceChunksToTake());
if (config.getTimeSeriesChunksToTake() != -Integer.MIN_VALUE)
setTimeSeriesChunksToTake(config.getTimeSeriesChunksToTake());
if (config.randomTake != null)
setRandomTake(config.isRandomTake());
if (config.useSimpleDistance != null)
setUseSimpleDistance(config.isUseSimpleDistance());
// database information merge
if (config.databaseDriver != null)
setDatabaseDriver(config.databaseDriver);
if (config.databaseDialect != null)
setDatabaseDialect(config.databaseDialect);
if (config.databaseAutomaticTestTable != null)
setDatabaseAutomaticTestTable(config.databaseAutomaticTestTable);
if (config.databaseIdleConnectionTestPeriod != null)
setDatabaseIdleConnectionTestPeriod(config.databaseIdleConnectionTestPeriod);
if (config.databaseUserName != null)
setDatabaseUserName(config.databaseUserName);
if (config.databasePassword != null)
setDatabasePassword(config.databasePassword);
if (config.databaseURL != null)
setDatabaseURL(config.databaseURL);
if (config.referenceTable != null)
setReferenceTable(config.referenceTable);
if (config.referenceColumn != null)
setReferenceColumn(config.referenceColumn);
if (config.idColumn != null)
setIdColumn(config.idColumn);
if (config.nameHuman != null)
setNameHuman(config.nameHuman);
if (config.description != null)
setDescription(config.description);
}
public void setDatabaseDriver(String databaseDriver) { public void setDatabaseDriver(String databaseDriver) {
this.databaseDriver = databaseDriver; this.databaseDriver = databaseDriver;
} }
public String getDatabaseDriver() { public String getDatabaseDriver() {
return databaseDriver; return databaseDriver;
} }
public void setDatabaseURL(String databaseURL) { public void setDatabaseURL(String databaseURL) {
this.databaseURL = databaseURL; this.databaseURL = databaseURL;
} }
public String getDatabaseURL() { public String getDatabaseURL() {
return databaseURL; return databaseURL;
} }
public void setDatabaseUserName(String databaseUserName) { public void setDatabaseUserName(String databaseUserName) {
this.databaseUserName = databaseUserName; this.databaseUserName = databaseUserName;
} }
public String getDatabaseUserName() { public String getDatabaseUserName() {
return databaseUserName; return databaseUserName;
} }
public void setDatabasePassword(String databasePassword) { public void setDatabasePassword(String databasePassword) {
this.databasePassword = databasePassword; this.databasePassword = databasePassword;
} }
public String getDatabasePassword() { public String getDatabasePassword() {
return databasePassword; return databasePassword;
} }
public void setDatabaseDialect(String databaseDialect) { public void setDatabaseDialect(String databaseDialect) {
this.databaseDialect = databaseDialect; this.databaseDialect = databaseDialect;
} }
public String getDatabaseDialect() { public String getDatabaseDialect() {
return databaseDialect; return databaseDialect;
} }
public void setDatabaseIdleConnectionTestPeriod(String databaseIdleConnectionTestPeriod) { public void setDatabaseIdleConnectionTestPeriod(String databaseIdleConnectionTestPeriod) {
this.databaseIdleConnectionTestPeriod = databaseIdleConnectionTestPeriod; this.databaseIdleConnectionTestPeriod = databaseIdleConnectionTestPeriod;
} }
public String getDatabaseIdleConnectionTestPeriod() { public String getDatabaseIdleConnectionTestPeriod() {
return databaseIdleConnectionTestPeriod; return databaseIdleConnectionTestPeriod;
} }
public void setDatabaseAutomaticTestTable(String databaseAutomaticTestTable) { public void setDatabaseAutomaticTestTable(String databaseAutomaticTestTable) {
this.databaseAutomaticTestTable = databaseAutomaticTestTable; this.databaseAutomaticTestTable = databaseAutomaticTestTable;
} }
public String getDatabaseAutomaticTestTable() { public String getDatabaseAutomaticTestTable() {
return databaseAutomaticTestTable; return databaseAutomaticTestTable;
} }
@ -289,32 +280,22 @@ public class LexicalEngineConfiguration {
return idColumn; return idColumn;
} }
public void setIdColumn(String idColumn) { public void setIdColumn(String idColumn) {
this.idColumn = idColumn; this.idColumn = idColumn;
} }
public String getNameHuman() { public String getNameHuman() {
return nameHuman; return nameHuman;
} }
public void setNameHuman(String nameHuman) { public void setNameHuman(String nameHuman) {
this.nameHuman = nameHuman; this.nameHuman = nameHuman;
} }
public String getDescription() { public String getDescription() {
return description; return description;
} }
public void setDescription(String description) { public void setDescription(String description) {
this.description = description; this.description = description;
} }

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example1_Species.java
View File

@ -13,7 +13,7 @@ public class Example1_Species {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example2_Area.java
View File

@ -13,7 +13,7 @@ public class Example2_Area {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example3_SingleMatchShark.java
View File

@ -15,7 +15,7 @@ public class Example3_SingleMatchShark {
try { try {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String singleton = "shark"; String singleton = "shark";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example4_SingleMatchMitella.java
View File

@ -15,7 +15,7 @@ public class Example4_SingleMatchMitella {
try { try {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String singleton = "Mitella pollicipes"; String singleton = "Mitella pollicipes";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/Example5_SingleMatchMitella.java
View File

@ -15,7 +15,7 @@ public class Example5_SingleMatchMitella {
try { try {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String singleton = "Mirella policepes"; String singleton = "Mirella policepes";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/examples/ExampleGuessingExternalCfg.java
View File

@ -13,7 +13,7 @@ public class ExampleGuessingExternalCfg {
try { try {
String configPath = "./"; String configPath = "./";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1

1
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/guesser/data/DBObjectTranslator.java
View File

@ -157,6 +157,7 @@ public class DBObjectTranslator {
} }
} }
public void buildCategories(SessionFactory dbSession, String referenceTable, String referenceColumn, String idColumn, String nameHuman, String description) { public void buildCategories(SessionFactory dbSession, String referenceTable, String referenceColumn, String idColumn, String nameHuman, String description) {
referenceTable = referenceTable == null ? "reference_table" : referenceTable; referenceTable = referenceTable == null ? "reference_table" : referenceTable;

59
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/run/CategoryGuesser.java
View File

@ -6,6 +6,7 @@ import java.util.ArrayList;
import org.gcube.contentmanagement.lexicalmatcher.analysis.core.Engine; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.Engine;
import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.CategoryOrderedList; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.CategoryOrderedList;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.DBObjectTranslator; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.DBObjectTranslator;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult;
@ -103,29 +104,23 @@ public class CategoryGuesser {
} }
// NOTE: The config path has to contain the two files: lexicalGuesser.properties and ALog.properties // NOTE: The config path has to contain the two files: lexicalGuesser.properties and ALog.properties
private static final String cfgFile = "lexicalGuesser.properties"; private static final String cfgFile = "lexicalguesser/lexicalGuesser.properties";
private static final String LogFile = "ALog.properties"; private static final String LogFile = "lexicalguesser/ALog.properties";
// singleton // singleton
private CategoryOrderedList col; private CategoryOrderedList col;
private Engine processor; private Engine processor;
private CategoryOrderedList originalCol; private CategoryOrderedList originalCol;
private LexicalEngineConfiguration config; private LexicalEngineConfiguration config;
private String configPath;
private boolean oneshotMode; private boolean oneshotMode;
private static final int maxTriesClassification = 3; private static final int maxTriesClassification = 3;
private int triesCounter; private int triesCounter;
public CategoryGuesser(String ConfigPath) {
triesCounter = 0;
this.configPath = ConfigPath;
}
public CategoryGuesser() { public CategoryGuesser() {
triesCounter = 0; triesCounter = 0;
this.configPath = ".";
} }
public void runGuesser(String seriesName, String columnName, LexicalEngineConfiguration externalConfig) throws Exception { public void runGuesser(String seriesName, String columnName, LexicalEngineConfiguration externalConfig) throws Exception {
runGuesser(seriesName, columnName, externalConfig, null, null); runGuesser(seriesName, columnName, externalConfig, null, null);
} }
@ -145,19 +140,19 @@ public class CategoryGuesser {
public void init(String categoryFilter, String columnFilter, LexicalEngineConfiguration externalConfig) throws Exception { public void init(String categoryFilter, String columnFilter, LexicalEngineConfiguration externalConfig) throws Exception {
String cfgFileCompletePath = configPath + "/" + cfgFile; String cfgFileCompletePath = cfgFile;
AnalysisLogger.setLogger(configPath + "/" + LogFile); AnalysisLogger.setLogger(LogFile);
AnalysisLogger.getLogger().trace("******************INITIALIZING******************"); AnalysisLogger.getLogger().trace("******************INITIALIZING******************");
config = new LexicalEngineConfiguration(); config = new LexicalEngineConfiguration();
config.configure(cfgFileCompletePath); config.configureByStream(cfgFileCompletePath);
if (externalConfig != null) { if (externalConfig != null) {
config.mergeConfig(externalConfig); config.mergeConfig(externalConfig);
} }
processor = new Engine(config, columnFilter, configPath); processor = new Engine(config, columnFilter);
SessionFactory dbSession = processor.getDBSession(config); SessionFactory dbSession = processor.getDBSession(config);
DBObjectTranslator dbo = new DBObjectTranslator(); DBObjectTranslator dbo = new DBObjectTranslator();
@ -166,7 +161,12 @@ public class CategoryGuesser {
AnalysisLogger.getLogger().trace("******************Order Category******************"); AnalysisLogger.getLogger().trace("******************Order Category******************");
if (externalConfig == null) if (externalConfig == null)
externalConfig = new LexicalEngineConfiguration(); externalConfig = new LexicalEngineConfiguration();
if (externalConfig.getCategories()!=null && externalConfig.getCategories().size()>0)
dbo.categories=externalConfig.getCategories();
else
dbo.buildCategoriesStructure(dbSession, externalConfig.getReferenceTable(), externalConfig.getReferenceColumn(), externalConfig.getIdColumn(), externalConfig.getNameHuman(), externalConfig.getDescription()); dbo.buildCategoriesStructure(dbSession, externalConfig.getReferenceTable(), externalConfig.getReferenceColumn(), externalConfig.getIdColumn(), externalConfig.getNameHuman(), externalConfig.getDescription());
col = TSObjectTransformer.transform2List(dbo, config, categoryFilter); col = TSObjectTransformer.transform2List(dbo, config, categoryFilter);
AnalysisLogger.getLogger().trace("***************End Ordering********************"); AnalysisLogger.getLogger().trace("***************End Ordering********************");
originalCol = col.generateNovelList(); originalCol = col.generateNovelList();
@ -179,17 +179,17 @@ public class CategoryGuesser {
public void initSingleMatcher(LexicalEngineConfiguration externalConfig, String ColumnFilter) throws Exception { public void initSingleMatcher(LexicalEngineConfiguration externalConfig, String ColumnFilter) throws Exception {
String cfgFileCompletePath = configPath + "/" + cfgFile; String cfgFileCompletePath = cfgFile;
AnalysisLogger.setLogger(configPath + "/" + LogFile); AnalysisLogger.setLogger(LogFile);
config = new LexicalEngineConfiguration(); config = new LexicalEngineConfiguration();
config.configure(cfgFileCompletePath); config.configureByStream(cfgFileCompletePath);
if (externalConfig != null) { if (externalConfig != null) {
config.mergeConfig(externalConfig); config.mergeConfig(externalConfig);
} }
processor = new Engine(config, ColumnFilter, configPath); processor = new Engine(config, ColumnFilter);
// in this case, the lexical matcher is invoked once, then it has to be stopped in the end // in this case, the lexical matcher is invoked once, then it has to be stopped in the end
oneshotMode = true; oneshotMode = true;
@ -213,17 +213,17 @@ public class CategoryGuesser {
public void runGuesser(String seriesName, String columnName, LexicalEngineConfiguration externalConfig, String CategoryFilter, String ColumnFilter, String SingletonString) throws Exception { public void runGuesser(String seriesName, String columnName, LexicalEngineConfiguration externalConfig, String CategoryFilter, String ColumnFilter, String SingletonString) throws Exception {
String cfgFileCompletePath = configPath + "/" + cfgFile; String cfgFileCompletePath = cfgFile;
AnalysisLogger.setLogger(configPath + "/" + LogFile); AnalysisLogger.setLogger(LogFile);
AnalysisLogger.getLogger().debug("Guessing Table " + seriesName + " column " + columnName); AnalysisLogger.getLogger().debug("Guessing Table " + seriesName + " column " + columnName);
if (externalConfig != null) { if (externalConfig != null) {
config = new LexicalEngineConfiguration(); config = new LexicalEngineConfiguration();
config.configure(cfgFileCompletePath); config.configureByStream(cfgFileCompletePath);
config.mergeConfig(externalConfig); config.mergeConfig(externalConfig);
// NOTE FOR FUTURE OPTIMIZATION: perform the re-init only if there is a change in the Database pointing // NOTE FOR FUTURE OPTIMIZATION: perform the re-init only if there is a change in the Database pointing
processor = new Engine(config, ColumnFilter, configPath); processor = new Engine(config, ColumnFilter);
} else { } else {
if (config == null) { if (config == null) {
config = new LexicalEngineConfiguration(); config = new LexicalEngineConfiguration();
@ -231,9 +231,9 @@ public class CategoryGuesser {
} }
if (processor == null) { if (processor == null) {
processor = new Engine(config, ColumnFilter, configPath); processor = new Engine(config, ColumnFilter);
} else } else
processor.resetEngine(config, ColumnFilter, configPath); processor.resetEngine(config, ColumnFilter);
} }
SessionFactory dbSession = processor.getDBSession(config); SessionFactory dbSession = processor.getDBSession(config);
@ -242,7 +242,16 @@ public class CategoryGuesser {
//modification of 10/10/11 calculate structure each time //modification of 10/10/11 calculate structure each time
// if (col == null) { // if (col == null) {
AnalysisLogger.getLogger().trace("******************Order Category******************"); AnalysisLogger.getLogger().trace("******************Order Category******************");
if (externalConfig !=null){
ArrayList<Category> externalcategories = externalConfig.getCategories();
if ((externalcategories!=null) && (externalcategories.size()>0)){
dbo.categories=externalConfig.getCategories();
dbo.calculateCategoriesWeights(dbSession);
}
else
dbo.buildCategoriesStructure(dbSession, config.getReferenceTable(), config.getReferenceColumn(), config.getIdColumn(), config.getNameHuman(), config.getDescription()); dbo.buildCategoriesStructure(dbSession, config.getReferenceTable(), config.getReferenceColumn(), config.getIdColumn(), config.getNameHuman(), config.getDescription());
}
col = TSObjectTransformer.transform2List(dbo, config, CategoryFilter); col = TSObjectTransformer.transform2List(dbo, config, CategoryFilter);
AnalysisLogger.getLogger().trace("***************End Ordering********************"); AnalysisLogger.getLogger().trace("***************End Ordering********************");
originalCol = col.generateNovelList(); originalCol = col.generateNovelList();
@ -277,7 +286,7 @@ public class CategoryGuesser {
config.setCategoryDiscardDifferencialThreshold(Math.max(differencialThr - 20, 0)); config.setCategoryDiscardDifferencialThreshold(Math.max(differencialThr - 20, 0));
config.setEntryAcceptanceThreshold(Math.max(acceptanceThr - 20, 0)); config.setEntryAcceptanceThreshold(Math.max(acceptanceThr - 20, 0));
AnalysisLogger.getLogger().trace("Performing next processing pass"); AnalysisLogger.getLogger().trace("Performing next processing pass");
runGuesser(seriesName, columnName, null, CategoryFilter, ColumnFilter, SingletonString); runGuesser(seriesName, columnName, externalConfig, CategoryFilter, ColumnFilter, SingletonString);
AnalysisLogger.getLogger().debug("End processing pass"); AnalysisLogger.getLogger().debug("End processing pass");
// if (oneshotMode) // if (oneshotMode)

20
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestExternalCfgProduction.java
View File

@ -1,6 +1,10 @@
package org.gcube.contentmanagement.lexicalmatcher.analysis.test; package org.gcube.contentmanagement.lexicalmatcher.analysis.test;
import java.util.ArrayList;
import java.util.Properties;
import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category;
import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
@ -11,9 +15,9 @@ public class TestExternalCfgProduction {
try { try {
int attempts = 1; int attempts = 1;
// new Properties().load(ClassLoader.getSystemResourceAsStream("lexicalguesser/lexicalGuesser.properties"));
String configPath = "."; CategoryGuesser guesser = new CategoryGuesser();
CategoryGuesser guesser = new CategoryGuesser(configPath);
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
@ -28,6 +32,18 @@ public class TestExternalCfgProduction {
conf.setDescription("ifield2"); conf.setDescription("ifield2");
ArrayList<Category> categories = new ArrayList<Category>();
//human name, index, table name, description
categories.add(new Category("COUNTRY_OLD","39c98800-dd3c-11e0-b8d1-d1e2e7ba4f9d","rdf39c98800dd3c11e0b8d1d1e2e7ba4f9d","country"));
categories.add(new Category("CONTINENT_OLD","1d5d51f0-dd42-11e0-b8d3-d1e2e7ba4f9d","rdf1d5d51f0dd4211e0b8d3d1e2e7ba4f9d","continent reference data"));
categories.add(new Category("SPECIES_OLD","0a7fb500-dd3d-11e0-b8d1-d1e2e7ba4f9d","rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d","species"));
categories.add(new Category("CodeListCountry","4c8d93a0-edc2-11e0-93e4-f6a9821baa29","rdf4c8d93a0edc211e093e4f6a9821baa29","Country"));
categories.add(new Category("CL_DIVISION","1140bdf0-dd2c-11e0-9220-ae17b3db32b7","rdf1140bdf0dd2c11e09220ae17b3db32b7","undefined"));
categories.add(new Category("CL_ASFIS_TAX","f87360f0-d9f9-11e0-ba05-d9adb0db767c","rdff87360f0d9f911e0ba05d9adb0db767c","undefined"));
conf.setCategories(categories);
//database Parameters //database Parameters
conf.setDatabaseUserName("gcube"); conf.setDatabaseUserName("gcube");
conf.setDatabasePassword("d4science2"); conf.setDatabasePassword("d4science2");

18
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/TestSingleExternalCfgProduction.java
View File

@ -1,8 +1,10 @@
package org.gcube.contentmanagement.lexicalmatcher.analysis.test; package org.gcube.contentmanagement.lexicalmatcher.analysis.test;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration; import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.Category;
import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult; import org.gcube.contentmanagement.lexicalmatcher.analysis.guesser.data.SingleResult;
import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser; import org.gcube.contentmanagement.lexicalmatcher.analysis.run.CategoryGuesser;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
@ -13,9 +15,7 @@ public class TestSingleExternalCfgProduction {
try { try {
CategoryGuesser guesser = new CategoryGuesser();
String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath);
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
@ -25,11 +25,23 @@ public class TestSingleExternalCfgProduction {
String column = "field6"; String column = "field6";
LexicalEngineConfiguration conf = new LexicalEngineConfiguration(); LexicalEngineConfiguration conf = new LexicalEngineConfiguration();
/*
conf.setReferenceTable("codelist1733371938"); conf.setReferenceTable("codelist1733371938");
conf.setReferenceColumn("ifield14"); conf.setReferenceColumn("ifield14");
conf.setNameHuman("ifield1"); conf.setNameHuman("ifield1");
conf.setIdColumn("ifield0"); conf.setIdColumn("ifield0");
conf.setDescription("ifield2"); conf.setDescription("ifield2");
*/
ArrayList<Category> categories = new ArrayList<Category>();
//human name, index, table name, description
categories.add(new Category("COUNTRY_OLD","39c98800-dd3c-11e0-b8d1-d1e2e7ba4f9d","rdf39c98800dd3c11e0b8d1d1e2e7ba4f9d","country"));
categories.add(new Category("CONTINENT_OLD","1d5d51f0-dd42-11e0-b8d3-d1e2e7ba4f9d","rdf1d5d51f0dd4211e0b8d3d1e2e7ba4f9d","continent reference data"));
categories.add(new Category("SPECIES_OLD","0a7fb500-dd3d-11e0-b8d1-d1e2e7ba4f9d","rdf0a7fb500dd3d11e0b8d1d1e2e7ba4f9d","species"));
categories.add(new Category("CodeListCountry","4c8d93a0-edc2-11e0-93e4-f6a9821baa29","rdf4c8d93a0edc211e093e4f6a9821baa29","Country"));
categories.add(new Category("CL_DIVISION","1140bdf0-dd2c-11e0-9220-ae17b3db32b7","rdf1140bdf0dd2c11e09220ae17b3db32b7","undefined"));
conf.setCategories(categories);
//CHANGE THIS TO ENHANCE THE RECALL //CHANGE THIS TO ENHANCE THE RECALL
conf.setEntryAcceptanceThreshold(30); conf.setEntryAcceptanceThreshold(30);

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest1.java
View File

@ -11,7 +11,7 @@ public class BenchMarkTest1 {
int attempts = 1; int attempts = 1;
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; String seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest2.java
View File

@ -12,7 +12,7 @@ public class BenchMarkTest2 {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest3.java
View File

@ -12,7 +12,7 @@ public class BenchMarkTest3 {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest4.java
View File

@ -12,7 +12,7 @@ public class BenchMarkTest4 {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6"; String seriesName = "import_2c97f580_35a0_11df_b8b3_aa10916debe6";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTest5.java
View File

@ -12,7 +12,7 @@ public class BenchMarkTest5 {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e"; String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestExternalCfg.java
View File

@ -13,7 +13,7 @@ public class BenchMarkTestExternalCfg {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e"; String seriesName = "import_532bba80_1c8f_11df_a4ee_87804054691e";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestFilterCategory.java
View File

@ -14,7 +14,7 @@ public class BenchMarkTestFilterCategory {
try { try {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "ref_order"; String seriesName = "ref_order";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestSingleton.java
View File

@ -15,7 +15,7 @@ public class BenchMarkTestSingleton {
try { try {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String singleton = "sarda sarda"; String singleton = "sarda sarda";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTestTSCountry.java
View File

@ -11,7 +11,7 @@ public class BenchMarkTestTSCountry {
int attempts = 1; int attempts = 1;
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592"; String seriesName = "import_bdefb470_5cea_11df_a0a6_909e7d074592";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSet.java
View File

@ -11,7 +11,7 @@ public static void main(String[] args) {
try { try {
String configPath ="."; String configPath =".";
int attempts = 1; int attempts = 1;
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/BenchMarkTrainingSetScientificName.java
View File

@ -11,7 +11,7 @@ public static void main(String[] args) {
try { try {
String configPath ="."; String configPath =".";
int attempts = 1; int attempts = 1;
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestExternalCfgProduction.java
View File

@ -13,7 +13,7 @@ public class TestExternalCfgProduction {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
// String seriesName = "rdmc366dfe0ddf511e086b1b1c5d6fb1c27"; // String seriesName = "rdmc366dfe0ddf511e086b1b1c5d6fb1c27";

2
src/main/java/org/gcube/contentmanagement/lexicalmatcher/analysis/test/old/TestSingleExternalCfgProduction.java
View File

@ -15,7 +15,7 @@ public class TestSingleExternalCfgProduction {
String configPath = "."; String configPath = ".";
CategoryGuesser guesser = new CategoryGuesser(configPath); CategoryGuesser guesser = new CategoryGuesser();
//bench 1 //bench 1
AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------"); AnalysisLogger.getLogger().warn("----------------------BENCH 1-------------------------");
String singleton = "Faroe Island"; String singleton = "Faroe Island";

15
src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/AnalysisLogger.java
View File

@ -1,5 +1,9 @@
package org.gcube.contentmanagement.lexicalmatcher.utils; package org.gcube.contentmanagement.lexicalmatcher.utils;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.PropertyConfigurator;
@ -25,7 +29,18 @@ public class AnalysisLogger {
//in ingresso vuole il path al file di config del log4j //in ingresso vuole il path al file di config del log4j
public static void setLogger(String path){ public static void setLogger(String path){
if (logger == null){ if (logger == null){
try{
PropertyConfigurator.configure(path); PropertyConfigurator.configure(path);
}catch(Exception e) {
Properties p = new Properties();
try {
InputStream is = ClassLoader.getSystemResourceAsStream(path);
p.load(is);
is.close();
} catch (IOException e1) {
}
PropertyConfigurator.configure(p);
}
} }
logger = Logger.getLogger("AnalysisLogger"); logger = Logger.getLogger("AnalysisLogger");
hibernateLogger = Logger.getLogger("hibernate"); hibernateLogger = Logger.getLogger("hibernate");

15
src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/DatabaseFactory.java
View File

@ -4,6 +4,7 @@ package org.gcube.contentmanagement.lexicalmatcher.utils;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -36,13 +37,16 @@ public class DatabaseFactory {
if (config==null) if (config==null)
return initDBConnection(configurationFile); return initDBConnection(configurationFile);
InputStream stream;
// take the configuration file try {
File fl = new File(configurationFile); File fl = new File(configurationFile);
FileInputStream stream = new FileInputStream(fl); stream = new FileInputStream(fl);
} catch (Exception e) {
stream = ClassLoader.getSystemResourceAsStream(configurationFile);
}
SAXReader saxReader = new SAXReader(); SAXReader saxReader = new SAXReader();
Document document = saxReader.read(stream); Document document = saxReader.read(stream);
List<Node> nodes = document.selectNodes("//hibernate-configuration/session-factory/property"); List<Node> nodes = document.selectNodes("//hibernate-configuration/session-factory/property");
Iterator<Node> nodesIterator = nodes.iterator(); Iterator<Node> nodesIterator = nodes.iterator();
@ -92,9 +96,6 @@ public class DatabaseFactory {
// close stream // close stream
stream.close(); stream.close();
return DBSessionFactory; return DBSessionFactory;
} }

11
src/main/java/org/gcube/contentmanagement/lexicalmatcher/utils/FileTools.java
View File

@ -6,6 +6,7 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
@ -18,12 +19,18 @@ public class FileTools {
public static String readXMLDoc(String xmlFilePath) throws Exception { public static String readXMLDoc(String xmlFilePath) throws Exception {
String xml = null; String xml = null;
InputStream stream;
try {
File fl = new File(xmlFilePath); File fl = new File(xmlFilePath);
FileInputStream stream = new FileInputStream(fl); stream = new FileInputStream(fl);
} catch (Exception e) {
stream = ClassLoader.getSystemResourceAsStream(xmlFilePath);
}
SAXReader saxReader = new SAXReader(); SAXReader saxReader = new SAXReader();
Document document = saxReader.read(stream); Document document = saxReader.read(stream);
xml = document.asXML(); xml = document.asXML();
stream.close();
return xml; return xml;
} }

7
src/main/java/org/gcube/dataanalysis/ecoengine/interfaces/DataAnalysis.java
View File

@ -97,10 +97,11 @@ public abstract class DataAnalysis implements Evaluator{
* visualizes the results of the analysis * visualizes the results of the analysis
* @param results * @param results
*/ */
public static void visualizeResults(HashMap<String,String> results){ public static void visualizeResults(HashMap<String,Object> results){
for (String key:results.keySet()){ for (Object key:results.keySet()){
System.out.println(key+":"+results.get(key)); PrimitiveType keyp = (PrimitiveType) results.get(key);
System.out.println(key+":"+keyp.getContent());
} }
} }

11
src/main/java/org/gcube/dataanalysis/ecoengine/test/TablesComparison.java
View File

@ -25,8 +25,12 @@ public class TablesComparison {
static double Threshold = 0.01; static double Threshold = 0.01;
//change this defaults to change comparison //change this defaults to change comparison
/*
public String referenceTable = "hspec_suitable_executor_1_worker"; public String referenceTable = "hspec_suitable_executor_1_worker";
public String analyzedTable = "hspec_suitable_executor_2"; public String analyzedTable = "hspec_suitable_executor_2";
*/
public String referenceTable = "hspec_suitable_latimeria_chalumnae";
public String analyzedTable = "hspec_suitable_neural_latimeria_chalumnae";
public String referenceCriteria = "speciesid,csquarecode"; public String referenceCriteria = "speciesid,csquarecode";
public String destinationCriteria = "speciesid,csquarecode"; public String destinationCriteria = "speciesid,csquarecode";
public String referenceSelectedColumns = "speciesid,csquarecode,probability"; public String referenceSelectedColumns = "speciesid,csquarecode,probability";
@ -84,9 +88,14 @@ public class TablesComparison {
String configPath = "./cfg/"; String configPath = "./cfg/";
AlgorithmConfiguration config = new AlgorithmConfiguration(); AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath(configPath); config.setConfigPath(configPath);
/*
config.setDatabaseUserName("utente"); config.setDatabaseUserName("utente");
config.setDatabasePassword("d4science"); config.setDatabasePassword("d4science");
config.setDatabaseURL("jdbc:postgresql://dbtest.research-infrastructures.eu/aquamapsorgupdated"); */
// config.setDatabaseURL("jdbc:postgresql://dbtest.research-infrastructures.eu/aquamapsorgupdated");
config.setDatabaseUserName("gcube");
config.setDatabasePassword("d4science2");
config.setDatabaseURL("jdbc:postgresql://localhost/testdb");
TablesComparison ec = new TablesComparison(config); TablesComparison ec = new TablesComparison(config);
long t0 = System.currentTimeMillis(); long t0 = System.currentTimeMillis();

180
src/main/java/org/gcube/dataanalysis/ecoengine/test/TablesComparisonLatimeria.java Normal file
View File

@ -0,0 +1,180 @@
package org.gcube.dataanalysis.ecoengine.test;
import java.math.BigInteger;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.contentmanagement.lexicalmatcher.utils.DatabaseFactory;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.hibernate.SessionFactory;
/**
* checks if two tables are equal
* checks numbers at the second decimal position
*/
public class TablesComparisonLatimeria {
private BigInteger numOfElements;
private int errorCounter;
//connection setup
protected String LogFile = "ALog.properties";
//fundamental: set a the chunk csquaresNumber and the maximum number of chunks to take
int chunkSize = 7000;
static double Threshold = 0.01;
//change this defaults to change comparison
public String referenceTable = "hspec_suitable_latimeria_chalumnae";
public String analyzedTable = "hspec_suitable_neural_latimeria_chalumnae";
public String referenceCriteria = "csquarecode";
public String destinationCriteria = "csquarecode";
public String referenceSelectedColumns = "csquarecode,probability";
public String destinationSelectedColumns = "csquarecode,probability";
//selection query
public static String selectElementsQuery = "select %1$s from %2$s order by %3$s";
//database connections
protected SessionFactory referencedbConnection;
protected SessionFactory destinationdbConnection;
//init connections
public TablesComparisonLatimeria(AlgorithmConfiguration config) throws Exception {
AnalysisLogger.setLogger(config.getConfigPath() + LogFile);
referencedbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config);
AnalysisLogger.getLogger().debug("ReferenceDB initialized");
destinationdbConnection = DatabaseFactory.initDBConnection(config.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile,config);
AnalysisLogger.getLogger().debug("OriginalDB initialized");
}
//counts the elements in a table
public BigInteger countElements(String tablename, SessionFactory session)
{
BigInteger count = BigInteger.ZERO;
String countingQuery = "select count(*) from "+tablename;
AnalysisLogger.getLogger().debug("Getting DB elements by this query: "+countingQuery);
List<Object> result = DatabaseFactory.executeSQLQuery(countingQuery, session);
count = (BigInteger) result.get(0);
return count;
}
//takes a chunk of elements from the database, belonging to the set of 170 selected species
public List<Object> takeChunkOfElements(String tablename,String selectedColumns,String criteria, int limit, int offset, SessionFactory session) {
String query = String.format(selectElementsQuery,selectedColumns,tablename,criteria)+ " limit " + limit + " offset " + offset;
AnalysisLogger.getLogger().debug("takeChunkOfElements-> executing query on DB: " + query);
List<Object> results = DatabaseFactory.executeSQLQuery(query, session);
return results;
}
//checks if a string is a number
public double isNumber(String element){
try{
double d = Double.parseDouble(element);
return d;
}catch(Exception e){
return -Double.MAX_VALUE;
}
}
public static void main(String[] args) throws Exception {
String configPath = "./cfg/";
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath(configPath);
/*
config.setDatabaseUserName("utente");
config.setDatabasePassword("d4science");
*/
// config.setDatabaseURL("jdbc:postgresql://dbtest.research-infrastructures.eu/aquamapsorgupdated");
config.setDatabaseUserName("postgres");
config.setDatabasePassword("d4science2");
config.setDatabaseURL("jdbc:postgresql://geoserver-dev.d4science-ii.research-infrastructures.eu/aquamapsdb");
TablesComparisonLatimeria ec = new TablesComparisonLatimeria(config);
long t0 = System.currentTimeMillis();
ec.runTest();
long t1 = System.currentTimeMillis();
float difference = (t1-t0);
difference = difference /(float)(1000*60);
System.out.println("Elapsed time : "+difference+" min");
}
//runs the test between the tables
public boolean runTest() {
long t0 = System.currentTimeMillis();
// take the number of elements
numOfElements = countElements(analyzedTable, destinationdbConnection);
AnalysisLogger.getLogger().debug("Remote DB contains " + numOfElements + " elements.");
int maxNumber = numOfElements.intValue();
int numOfChunks = maxNumber / chunkSize;
if ((maxNumber % chunkSize) > 0) {
numOfChunks++;
}
int startIndex = 0;
// reset error counter
errorCounter = 0;
boolean equal = true;
for (int i = startIndex; i < numOfChunks; i++) {
AnalysisLogger.getLogger().debug("Chunk "+(i+1)+" of "+numOfChunks);
int offset = i * chunkSize;
List<Object> referencechunk = takeChunkOfElements(referenceTable,referenceSelectedColumns,referenceCriteria, chunkSize, offset, referencedbConnection);
List<Object> destinationchunk = takeChunkOfElements(analyzedTable,destinationSelectedColumns,destinationCriteria, chunkSize, offset, destinationdbConnection);
int m = referencechunk.size();
for (int j=0;j<m;j++){
Object[] refrow = (Object[]) referencechunk.get(j);
Object[] destrow = (Object[]) destinationchunk.get(j);
int columns = destrow.length;
for (int k=0;k<columns;k++){
String refelem = ""+refrow[k];
String destelem = ""+destrow[k];
double d = isNumber(refelem);
// System.out.print(refelem+" vs "+destelem+ " ");
if (d!=-Double.MAX_VALUE){
if (Math.abs(d-isNumber(destelem))>Threshold){
errorCounter++;
equal = false;
AnalysisLogger.getLogger().debug("ERROR - DISCREPANCY AT NUMBERS COMPARISON: "+refelem+" vs "+destelem);
}
}
else if (!refelem.equals(destelem)){
errorCounter++;
equal = false;
AnalysisLogger.getLogger().debug("ERROR - DISCREPANCY AT STRING COMPARISON: "+refelem+" vs "+destelem);
}
if (!equal)
break;
}
// System.out.println();
if (!equal)
break;
}
if (!equal)
break;
else
AnalysisLogger.getLogger().debug("CHUNK NUMBER "+i+" OK!");
}
long t1 = System.currentTimeMillis();
AnalysisLogger.getLogger().debug("ELAPSED TIME: " + (t1-t0) + " ms");
//close connections
referencedbConnection.close();
destinationdbConnection.close();
return equal;
}
}

2
src/main/java/org/gcube/dataanalysis/ecoengine/test/checks/TestEvaluation.java
View File

@ -77,7 +77,7 @@ public static void main(String[] args) throws Exception {
try { try {
dg.compute(); dg.compute();
PrimitiveType output = (PrimitiveType) dg.getOutput(); PrimitiveType output = (PrimitiveType) dg.getOutput();
HashMap<String, String> out = (HashMap<String, String>)output.getContent(); HashMap<String, Object> out = (HashMap<String, Object>)output.getContent();
DiscrepancyAnalysis.visualizeResults(out); DiscrepancyAnalysis.visualizeResults(out);
} catch (Exception e) { } catch (Exception e) {

6
src/main/java/org/gcube/dataanalysis/ecoengine/test/experiments/latimeria/ExperimentsForLatimeria.java
View File

@ -191,7 +191,7 @@ public class ExperimentsForLatimeria {
evaluators.get(0).init(); evaluators.get(0).init();
Regressor.process(evaluators.get(0)); Regressor.process(evaluators.get(0));
PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput(); PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput();
HashMap<String, String> out = (HashMap<String, String>)output.getContent(); HashMap<String, Object> out = (HashMap<String, Object>)output.getContent();
DiscrepancyAnalysis.visualizeResults(out); DiscrepancyAnalysis.visualizeResults(out);
evaluators = null; evaluators = null;
} }
@ -203,7 +203,7 @@ public class ExperimentsForLatimeria {
evaluators.get(0).init(); evaluators.get(0).init();
Regressor.process(evaluators.get(0)); Regressor.process(evaluators.get(0));
PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput(); PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput();
HashMap<String, String> out = (HashMap<String, String>)output.getContent(); HashMap<String, Object> out = (HashMap<String, Object>)output.getContent();
DiscrepancyAnalysis.visualizeResults(out); DiscrepancyAnalysis.visualizeResults(out);
evaluators = null; evaluators = null;
} }
@ -225,7 +225,7 @@ public class ExperimentsForLatimeria {
evaluators.get(0).init(); evaluators.get(0).init();
Regressor.process(evaluators.get(0)); Regressor.process(evaluators.get(0));
PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput(); PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput();
HashMap<String, String> out = (HashMap<String, String>)output.getContent(); HashMap<String, Object> out = (HashMap<String, Object>)output.getContent();
DiscrepancyAnalysis.visualizeResults(out); DiscrepancyAnalysis.visualizeResults(out);
evaluators = null; evaluators = null;
} }

111
src/main/java/org/gcube/dataanalysis/ecoengine/test/experiments/latimeria/TablesComparisonForLatimeria.java Normal file
View File

@ -0,0 +1,111 @@
package org.gcube.dataanalysis.ecoengine.test.experiments.latimeria;
import java.util.HashMap;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.evaluation.DiscrepancyAnalysis;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator;
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
import org.gcube.dataanalysis.ecoengine.interfaces.Modeler;
import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer;
import org.gcube.dataanalysis.ecoengine.processing.factories.EvaluatorsFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
import org.gcube.dataanalysis.ecoengine.utils.PresetConfigGenerator;
public class TablesComparisonForLatimeria {
static String absenceRandomTable = "absence_data_latimeria_random";
static String absenceStaticTable = "absence_data_latimeria";
static String presenceTable = "presence_data_latimeria_2";
static String presenceTableNoEarth = "presence_data_latimeria_sea";
static String envelopeTable = "hspen_latimeria";
static String aquamapsSuitableTable = "hspec_suitable_latimeria_chalumnae";
static String aquamapsNativeTable = "hspec_native_latimeria_chalumnae";
static String nnsuitableTable = "hspec_suitable_neural_latimeria_chalumnae";
static String nnsuitableRandomTable = "hspec_suitable_neural_latimeria_chalumnae_random";
static String nnnativeTable = "hspec_native_neural_latimeria_chalumnae";
static String nnnativeRandomTable = "hspec_native_neural_latimeria_chalumnae_random";
static String hcaf = "hcaf_d";
static String filteredhcaf = "bboxed_hcaf_d";
static String speciesID = "Fis-30189";
static String staticsuitable = "staticsuitable";
static String randomsuitable = "randomsuitable";
static String staticnative = "staticnative";
static String randomnative = "randomnative";
static int numberOfPoints = 34;
static String nnname = "neuralname";
static float x1 = 95.346678f;
static float y1 = -9.18887f;
static float x2 = 125.668944f;
static float y2 = 12.983148f;
public static AlgorithmConfiguration configDiscrepancyAnalysis(String table1, String table2) {
AlgorithmConfiguration config = getConfig();
config.setNumberOfResources(1);
config.setAgent("DISCREPANCY_ANALYSIS");
config.setParam("FirstTable", table1);
config.setParam("SecondTable", table2);
config.setParam("FirstTableCsquareColumn", "csquarecode");
config.setParam("SecondTableCsquareColumn", "csquarecode");
config.setParam("FirstTableProbabilityColumn", "probability");
config.setParam("SecondTableProbabilityColumn", "probability");
config.setParam("ComparisonThreshold", "0.1");
return config;
}
public static AlgorithmConfiguration getConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "postgres");
config.setParam("DatabasePassword", "d4science2");
config.setParam("DatabaseURL", "jdbc:postgresql://geoserver-dev.d4science-ii.research-infrastructures.eu/aquamapsdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
return config;
}
public static void calcdiscrepancy(String table1, String table2) throws Exception {
System.out.println("*****************************DISCREPANCY: " + table1 + " vs " + table2 + "************************************");
List<ComputationalAgent> evaluators = null;
evaluators = EvaluatorsFactory.getEvaluators(configDiscrepancyAnalysis(table1, table2));
evaluators.get(0).init();
Regressor.process(evaluators.get(0));
PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput();
HashMap<String, Object> out = (HashMap<String, Object>) output.getContent();
DiscrepancyAnalysis.visualizeResults(out);
evaluators = null;
}
public static void calcquality(String table, String presenceTable, String absenceTable) throws Exception {
System.out.println("*****************************QUALITY: " + table + " vs " + presenceTable + " and " + absenceTable + "************************************");
List<ComputationalAgent> evaluators = null;
evaluators = EvaluatorsFactory.getEvaluators(PresetConfigGenerator.configQualityAnalysis(presenceTable, absenceTable, table));
evaluators.get(0).init();
Regressor.process(evaluators.get(0));
PrimitiveType output = (PrimitiveType) evaluators.get(0).getOutput();
HashMap<String, Object> out = (HashMap<String, Object>) output.getContent();
DiscrepancyAnalysis.visualizeResults(out);
evaluators = null;
}
public static void main(String[] args) throws Exception {
calcdiscrepancy(aquamapsSuitableTable, nnsuitableTable);
calcdiscrepancy(aquamapsNativeTable, nnnativeTable);
}
}

4
src/main/java/org/gcube/dataanalysis/ecoengine/test/regression/Regressor.java
View File

@ -18,11 +18,11 @@ public class Regressor {
String resLoad = agent.getResourceLoad(); String resLoad = agent.getResourceLoad();
String ress = agent.getResources(); String ress = agent.getResources();
/*
System.out.println("LOAD: " + resLoad); System.out.println("LOAD: " + resLoad);
System.out.println("RESOURCES: " + ress); System.out.println("RESOURCES: " + ress);
System.out.println("STATUS: " + agent.getStatus()); System.out.println("STATUS: " + agent.getStatus());
*/
Thread.sleep(10000); Thread.sleep(10000);
} }
} else } else

2
src/main/java/org/gcube/dataanalysis/ecoengine/user/EvaluatorT.java
View File

@ -20,7 +20,7 @@ public class EvaluatorT implements Runnable{
try { try {
dg.compute(); dg.compute();
PrimitiveType output = (PrimitiveType) dg.getOutput(); PrimitiveType output = (PrimitiveType) dg.getOutput();
HashMap<String, String> out = (HashMap<String, String>)output.getContent(); HashMap<String, Object> out = (HashMap<String, Object>)output.getContent();
DiscrepancyAnalysis.visualizeResults(out); DiscrepancyAnalysis.visualizeResults(out);
} catch (Exception e) { } catch (Exception e) {

77
src/main/java/org/gcube/dataanalysis/ecoengine/utils/PresetConfigGenerator.java
View File

@ -1,17 +1,17 @@
package org.gcube.dataanalysis.ecoengine.utils; package org.gcube.dataanalysis.ecoengine.utils;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class PresetConfigGenerator { public class PresetConfigGenerator {
public static AlgorithmConfiguration configAquamapsSuitable(String aquamapsSuitableTable, String envelopeTable) { public static AlgorithmConfiguration configAquamapsSuitable(String aquamapsSuitableTable, String envelopeTable) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(5); config.setNumberOfResources(5);
config.setModel("AQUAMAPS_SUITABLE"); config.setModel("AQUAMAPS_SUITABLE");
config.setParam("DistributionTable",aquamapsSuitableTable ); config.setParam("DistributionTable", aquamapsSuitableTable);
config.setParam("CsquarecodesTable", "hcaf_d"); config.setParam("CsquarecodesTable", "hcaf_d");
config.setParam("EnvelopeTable", envelopeTable); config.setParam("EnvelopeTable", envelopeTable);
config.setParam("OccurrencePointsTable", "occurrencecells"); config.setParam("OccurrencePointsTable", "occurrencecells");
@ -36,7 +36,7 @@ public class PresetConfigGenerator {
return config; return config;
} }
public static AlgorithmConfiguration configAquamapsNNSuitable(String tableName,String username, String envelopeTable, String speciesID,String nnname) { public static AlgorithmConfiguration configAquamapsNNSuitable(String tableName, String username, String envelopeTable, String speciesID, String nnname) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(5); config.setNumberOfResources(5);
@ -53,7 +53,7 @@ public class PresetConfigGenerator {
return config; return config;
} }
public static AlgorithmConfiguration configAquamapsNNNative(String tableName,String username, String envelopeTable, String speciesID,String nnname) { public static AlgorithmConfiguration configAquamapsNNNative(String tableName, String username, String envelopeTable, String speciesID, String nnname) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(5); config.setNumberOfResources(5);
@ -69,7 +69,7 @@ public class PresetConfigGenerator {
return config; return config;
} }
public static AlgorithmConfiguration configSuitableNeuralNetworkTraining(String presenceTable, String absenceTable,String username, String speciesID, String neuronsAndLayers,String nnname) { public static AlgorithmConfiguration configSuitableNeuralNetworkTraining(String presenceTable, String absenceTable, String username, String speciesID, String neuronsAndLayers, String nnname) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(5); config.setNumberOfResources(5);
@ -78,13 +78,13 @@ public class PresetConfigGenerator {
config.setParam("AbsenceDataTable", absenceTable); config.setParam("AbsenceDataTable", absenceTable);
config.setParam("PresenceDataTable", presenceTable); config.setParam("PresenceDataTable", presenceTable);
config.setParam("SpeciesName", speciesID); config.setParam("SpeciesName", speciesID);
config.setParam("UserName",username); config.setParam("UserName", username);
config.setParam("LayersNeurons", neuronsAndLayers); config.setParam("LayersNeurons", neuronsAndLayers);
config.setParam("NeuralNetworkName", nnname); config.setParam("NeuralNetworkName", nnname);
return config; return config;
} }
public static AlgorithmConfiguration configNativeNeuralNetworkTraining(String presenceTable, String absenceTable,String username, String speciesID, String neuronsAndLayers,String nnname) { public static AlgorithmConfiguration configNativeNeuralNetworkTraining(String presenceTable, String absenceTable, String username, String speciesID, String neuronsAndLayers, String nnname) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(5); config.setNumberOfResources(5);
@ -100,71 +100,70 @@ public class PresetConfigGenerator {
return config; return config;
} }
public static AlgorithmConfiguration configQualityAnalysis(String presenceTable, String absenceTable, String table) {
public static AlgorithmConfiguration configQualityAnalysis(String presenceTable, String absenceTable, String table){
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);
config.setAgent("QUALITY_ANALYSIS"); config.setAgent("QUALITY_ANALYSIS");
config.setParam("PositiveCasesTable",presenceTable); config.setParam("PositiveCasesTable", presenceTable);
config.setParam("NegativeCasesTable",absenceTable); config.setParam("NegativeCasesTable", absenceTable);
config.setParam("PositiveCasesTableKeyColumn","csquarecode"); config.setParam("PositiveCasesTableKeyColumn", "csquarecode");
config.setParam("NegativeCasesTableKeyColumn","csquarecode"); config.setParam("NegativeCasesTableKeyColumn", "csquarecode");
config.setParam("DistributionTable",table); config.setParam("DistributionTable", table);
config.setParam("DistributionTableKeyColumn","csquarecode"); config.setParam("DistributionTableKeyColumn", "csquarecode");
config.setParam("DistributionTableProbabilityColumn","probability"); config.setParam("DistributionTableProbabilityColumn", "probability");
config.setParam("PositiveThreshold","0.8"); config.setParam("PositiveThreshold", "0.8");
config.setParam("NegativeThreshold","0.3"); config.setParam("NegativeThreshold", "0.3");
return config; return config;
} }
public static AlgorithmConfiguration configDiscrepancyAnalysis(String table1,String table2){ public static AlgorithmConfiguration configDiscrepancyAnalysis(String table1, String table2) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);
config.setAgent("DISCREPANCY_ANALYSIS"); config.setAgent("DISCREPANCY_ANALYSIS");
config.setParam("FirstTable",table1); config.setParam("FirstTable", table1);
config.setParam("SecondTable",table2); config.setParam("SecondTable", table2);
config.setParam("FirstTableCsquareColumn","csquarecode"); config.setParam("FirstTableCsquareColumn", "csquarecode");
config.setParam("SecondTableCsquareColumn","csquarecode"); config.setParam("SecondTableCsquareColumn", "csquarecode");
config.setParam("FirstTableProbabilityColumn","probability"); config.setParam("FirstTableProbabilityColumn", "probability");
config.setParam("SecondTableProbabilityColumn","probability"); config.setParam("SecondTableProbabilityColumn", "probability");
config.setParam("ComparisonThreshold","0.1"); config.setParam("ComparisonThreshold", "0.1");
return config; return config;
} }
public static AlgorithmConfiguration configHRSAnalysis(String projectiontable,String absenceTable, String presenceTable){ public static AlgorithmConfiguration configHRSAnalysis(String projectiontable, String absenceTable, String presenceTable) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);
config.setAgent("HRS"); config.setAgent("HRS");
config.setParam("ProjectingAreaTable", projectiontable); config.setParam("ProjectingAreaTable", projectiontable);
config.setParam("ProjectingAreaFeaturesOptionalCondition", "where oceanarea>0"); config.setParam("ProjectingAreaFeaturesOptionalCondition", "where oceanarea>0");
config.setParam("FeaturesColumns", "depthmean"+AlgorithmConfiguration.getListSeparator()+"depthmax"+AlgorithmConfiguration.getListSeparator()+"depthmin"+AlgorithmConfiguration.getListSeparator()+" sstanmean"+AlgorithmConfiguration.getListSeparator()+"sbtanmean"+AlgorithmConfiguration.getListSeparator()+"salinitymean"+AlgorithmConfiguration.getListSeparator()+"salinitybmean"+AlgorithmConfiguration.getListSeparator()+" primprodmean"+AlgorithmConfiguration.getListSeparator()+"iceconann"+AlgorithmConfiguration.getListSeparator()+"landdist"+AlgorithmConfiguration.getListSeparator()+"oceanarea"); config.setParam("FeaturesColumns", "depthmean" + AlgorithmConfiguration.getListSeparator() + "depthmax" + AlgorithmConfiguration.getListSeparator() + "depthmin" + AlgorithmConfiguration.getListSeparator() + " sstanmean" + AlgorithmConfiguration.getListSeparator() + "sbtanmean" + AlgorithmConfiguration.getListSeparator() + "salinitymean" + AlgorithmConfiguration.getListSeparator() + "salinitybmean" + AlgorithmConfiguration.getListSeparator() + " primprodmean" + AlgorithmConfiguration.getListSeparator() + "iceconann" + AlgorithmConfiguration.getListSeparator() + "landdist" + AlgorithmConfiguration.getListSeparator() + "oceanarea");
config.setParam("PositiveCasesTable", presenceTable); config.setParam("PositiveCasesTable", presenceTable);
config.setParam("NegativeCasesTable", absenceTable); config.setParam("NegativeCasesTable", absenceTable);
return config; return config;
} }
public static AlgorithmConfiguration configAbsenceTable(boolean random, String absenceTable, String hspecTable, int numberOfPoints, String speciesCode){ public static AlgorithmConfiguration configAbsenceTable(boolean random, String absenceTable, String hspecTable, int numberOfPoints, String speciesCode) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);
config.setAgent("ABSENCE_CELLS_FROM_AQUAMAPS"); config.setAgent("ABSENCE_CELLS_FROM_AQUAMAPS");
config.setParam("RANDOM_TAKE", ""+random); config.setParam("RANDOM_TAKE", "" + random);
config.setParam("FINAL_TABLE_NAME", absenceTable); config.setParam("FINAL_TABLE_NAME", absenceTable);
config.setParam("AQUAMAPS_HSPEC", hspecTable); config.setParam("AQUAMAPS_HSPEC", hspecTable);
config.setParam("SPECIES_CODE", speciesCode); config.setParam("SPECIES_CODE", speciesCode);
config.setParam("NUMBER_OF_POINTS", ""+numberOfPoints); config.setParam("NUMBER_OF_POINTS", "" + numberOfPoints);
return config; return config;
} }
public static AlgorithmConfiguration configPresenceTable(String presenceTable, int numberOfPoints, String speciesCode){ public static AlgorithmConfiguration configPresenceTable(String presenceTable, int numberOfPoints, String speciesCode) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);
@ -172,12 +171,12 @@ public class PresetConfigGenerator {
config.setParam("Table_Label", presenceTable); config.setParam("Table_Label", presenceTable);
config.setParam("Table_Name", presenceTable); config.setParam("Table_Name", presenceTable);
config.setParam("Species_Code", speciesCode); config.setParam("Species_Code", speciesCode);
config.setParam("Number_of_Points", ""+numberOfPoints); config.setParam("Number_of_Points", "" + numberOfPoints);
return config; return config;
} }
public static AlgorithmConfiguration configHCAFfilter(String table, float x1,float y1, float x2, float y2){ public static AlgorithmConfiguration configHCAFfilter(String table, float x1, float y1, float x2, float y2) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);
@ -185,15 +184,15 @@ public class PresetConfigGenerator {
config.setParam("FINAL_TABLE_NAME", table); config.setParam("FINAL_TABLE_NAME", table);
config.setParam("BOUNDING_BOX_LEFT_LOWER_LONG", ""+x1); config.setParam("BOUNDING_BOX_LEFT_LOWER_LONG", "" + x1);
config.setParam("BOUNDING_BOX_RIGHT_UPPER_LONG", ""+x2); config.setParam("BOUNDING_BOX_RIGHT_UPPER_LONG", "" + x2);
config.setParam("BOUNDING_BOX_LEFT_LOWER_LAT", ""+y1); config.setParam("BOUNDING_BOX_LEFT_LOWER_LAT", "" + y1);
config.setParam("BOUNDING_BOX_RIGHT_UPPER_LAT", ""+y2); config.setParam("BOUNDING_BOX_RIGHT_UPPER_LAT", "" + y2);
return config; return config;
} }
public static AlgorithmConfiguration configHSPENfilter(String table, String speciesList){ public static AlgorithmConfiguration configHSPENfilter(String table, String speciesList) {
AlgorithmConfiguration config = Regressor.getConfig(); AlgorithmConfiguration config = Regressor.getConfig();
config.setNumberOfResources(1); config.setNumberOfResources(1);

15
src/main/resources/lexicalguesser/ALog.properties Normal file
View File

@ -0,0 +1,15 @@
# Use two appenders, one to log to console, another to log to a file
#log4j.rootLogger=AR
#### Second appender writes to a file
log4j.logger.AnalysisLogger=TRACE,AR
log4j.appender.AR=org.apache.log4j.RollingFileAppender
log4j.appender.AR.Threshold=TRACE
log4j.appender.AR.File=${GLOBUS_LOCATION}/logs/Analysis.log
log4j.appender.AR.MaxFileSize=1024KB
log4j.appender.AR.MaxBackupIndex=2
log4j.appender.AR.layout=org.apache.log4j.PatternLayout
log4j.appender.AR.layout.ConversionPattern=%d{dd/MM/yyyy HH:mm:ss} %p %t %c - %m%n
#### Third appender writes to a file
log4j.logger.org.hibernate = fatal

18
src/main/resources/lexicalguesser/hibernate.cfg.xml Normal file
View File

@ -0,0 +1,18 @@
<?xml version='1.0' encoding='UTF-8'?>
<hibernate-configuration>
<session-factory>
<property name="connection.driver_class">org.postgresql.Driver</property>
<property name="connection.provider_class">org.hibernate.connection.C3P0ConnectionProvider</property>
<property name="connection.url">jdbc:postgresql://localhost/testdb</property>
<property name="connection.username">gcube</property>
<property name="connection.password">d4science2</property>
<!-- <property name="dialect">org.hibernatespatial.postgis.PostgisDialect</property>-->
<property name="dialect">org.hibernate.dialect.PostgreSQLDialect</property>
<property name="transaction.factory_class">org.hibernate.transaction.JDBCTransactionFactory</property>
<property name="c3p0.timeout">0</property>
<property name="c3p0.max_size">1</property>
<property name="c3p0.max_statements">0</property>
<property name="c3p0.min_size">1</property>
<property name="current_session_context_class">thread</property>
</session-factory>
</hibernate-configuration>

20
src/main/resources/lexicalguesser/lexicalGuesser.properties Normal file
View File

@ -0,0 +1,20 @@
#Percentage threshold for discarding a category
categoryDiscardThreshold=0
#Percentage threshold for accepting similarity between a single Time series entry and a reference entry
entryAcceptanceThreshold=80
#Size of a comparison chunk
chunkSize=25
#Number of chunks to take from Time series for performing comparison respect to reference data; if set to -1 all chunks will be analyzed
timeSeriesChunksToTake=2
#Number of chunks to take from Reference for performing comparison Time Series Elements; if set to -1 all chunks will be analyzed
referenceChunksToTake =50
#Use random choice for chunks selection = true |false
randomTake=true
#Use Simple String Match as distance calculation
useSimpleDistance=false
#Number Of Threads to use
numberOfThreadsToUse=5
#if two final scores differ for more than this percentage, prune the lower result
categoryDiscardDifferencialThreshold = 2
#maximum difference between a result and the best result
singleEntryRecognitionMaxDeviation = 40

BIN
target/ecological-engine-1.6.0-SNAPSHOT.jar
View File

Binary file not shown.