Implemented Neural Networks with trainers and testers
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@49905 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
050f05795c
commit
be22b91c3f
|
|
@ -20,5 +20,6 @@
|
|||
<classpathentry kind="lib" path="/StatisticalLibSupportLibraries/lib/postgresql-8.4-702.jdbc4.jar"/>
|
||||
<classpathentry kind="lib" path="/StatisticalLibSupportLibraries/lib/gson-1.7.1.jar"/>
|
||||
<classpathentry kind="lib" path="/StatisticalLibSupportLibraries/lib/StatisticsExtractor.jar"/>
|
||||
<classpathentry combineaccessrules="false" kind="src" path="/RapidMiner_Wasat"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -7,4 +7,7 @@ REMOTE_AQUAMAPS_NATIVE=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGen
|
|||
REMOTE_AQUAMAPS_NATIVE_2050=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator
|
||||
REMOTE_AQUAMAPS_SUITABLE_2050=org.gcube.dataanalysis.ecoengine.processing.RainyCloudGenerator
|
||||
DUMMY=org.gcube.dataanalysis.ecoengine.spatialdistributions.DummyAlgorithm
|
||||
TEST=org.gcube.dataanalysis.ecoengine.spatialdistributions.TestAlgorithm
|
||||
TEST=org.gcube.dataanalysis.ecoengine.spatialdistributions.TestAlgorithm
|
||||
AQUAMAPS_NATIVE_NEURALNETWORK=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNN
|
||||
AQUAMAPS_SUITABLE_NEURALNETWORK=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNNSuitable
|
||||
AQUAMAPS_NEURAL_NETWORK_NS=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNNNS
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
DISCREPANCY_ANALYSIS=org.gcube.dataanalysis.ecoengine.evaluation.DiscrepancyAnalysis
|
||||
QUALITY_ANALYSIS=org.gcube.dataanalysis.ecoengine.evaluation.DistributionQualityAnalysis
|
||||
|
|
@ -1 +1,3 @@
|
|||
HSPEN=org.gcube.dataanalysis.ecoengine.models.ModelHSPEN
|
||||
AQUAMAPSNN=org.gcube.dataanalysis.ecoengine.models.ModelAquamapsNN
|
||||
AQUAMAPSNNNS=org.gcube.dataanalysis.ecoengine.models.ModelAquamapsNNNS
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -6,5 +6,6 @@ public enum ALG_PROPS {
|
|||
SPECIES_VS_CSQUARE_REMOTE_FROM_DATABASE,
|
||||
SPECIES_VS_CSQUARE,
|
||||
PHENOMENON_VS_GEOINFO,
|
||||
SPECIES_ENVELOPES
|
||||
SPECIES_ENVELOPES,
|
||||
SPECIES_MODEL
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ import java.util.List;
|
|||
import java.util.Properties;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.analysis.core.LexicalEngineConfiguration;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class AlgorithmConfiguration extends LexicalEngineConfiguration{
|
||||
|
||||
|
|
@ -33,6 +36,7 @@ public class AlgorithmConfiguration extends LexicalEngineConfiguration{
|
|||
public static String generatorsFile = "generators.properties";
|
||||
public static String modelsFile = "models.properties";
|
||||
public static String modelersFile = "modelers.properties";
|
||||
public static String evaluatorsFile = "evaluators.properties";
|
||||
public static int chunkSize = 100000;
|
||||
public static int refreshResourcesTime = 10;
|
||||
|
||||
|
|
@ -134,12 +138,30 @@ public class AlgorithmConfiguration extends LexicalEngineConfiguration{
|
|||
this.persistencePath = persistencePath;
|
||||
}
|
||||
|
||||
public String getGenerator() {
|
||||
public String getAgent() {
|
||||
return generator;
|
||||
}
|
||||
|
||||
public void setGenerator(String generator) {
|
||||
public void setAgent(String generator) {
|
||||
this.generator = generator;
|
||||
}
|
||||
|
||||
public static SessionFactory getConnectionFromConfig(AlgorithmConfiguration Input){
|
||||
// init the database
|
||||
String defaultDatabaseFile = Input.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile;
|
||||
|
||||
Input.setDatabaseDriver(Input.getParam("DatabaseDriver"));
|
||||
Input.setDatabaseUserName(Input.getParam("DatabaseUserName"));
|
||||
Input.setDatabasePassword(Input.getParam("DatabasePassword"));
|
||||
Input.setDatabaseURL(Input.getParam("DatabaseURL"));
|
||||
SessionFactory connection = null;
|
||||
try {
|
||||
connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, Input);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().trace("ERROR initializing connection");
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
package org.gcube.dataanalysis.ecoengine.evaluation;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class DiscrepancyAnalysis extends DataAnalysis {
|
||||
|
||||
static String discrepancyQuery = "select distinct a.%1$s as csquareone,b.%2$s as csquaretwo,a.%3$s as firstprob,b.%4$s as secondprob from %5$s as a inner join %6$s as b on a.%1$s=b.%2$s and (a.%3$s<>b.%4$s)";
|
||||
static String getNumberOfElementsQuery = "select count(*) from %1$s";
|
||||
|
||||
float threshold = 0.1f;
|
||||
String configPath = "./cfg/";
|
||||
SessionFactory connection;
|
||||
List<Float> errors;
|
||||
double mean;
|
||||
double variance;
|
||||
int numberoferrors;
|
||||
int numberofvectors;
|
||||
float maxerror;
|
||||
String maxdiscrepancyPoint;
|
||||
|
||||
@Override
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
parameters.put("FirstTable", new VarCouple(VARTYPE.STRING, "hspec1"));
|
||||
parameters.put("SecondTable", new VarCouple(VARTYPE.STRING, "hspec2"));
|
||||
parameters.put("FirstTableCsquareColumn", new VarCouple(VARTYPE.STRING, "csquare"));
|
||||
parameters.put("SecondTableCsquareColumn", new VarCouple(VARTYPE.STRING, "csquarecode"));
|
||||
parameters.put("FirstTableProbabilityColumn", new VarCouple(VARTYPE.STRING, "csquare"));
|
||||
parameters.put("SecondTableProbabilityColumn", new VarCouple(VARTYPE.STRING, "csquarecode"));
|
||||
parameters.put("ComparisonThreshold", new VarCouple(VARTYPE.STRING, "0.1"));
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getOutputParameters() {
|
||||
|
||||
List<String> outputs = new ArrayList<String>();
|
||||
|
||||
outputs.add("MEAN");
|
||||
outputs.add("VARIANCE");
|
||||
outputs.add("NUMBER_OF_ERRORS");
|
||||
outputs.add("NUMBER_OF_COMPARISONS");
|
||||
outputs.add("ACCURACY");
|
||||
outputs.add("MAXIMUM_ERROR");
|
||||
outputs.add("MAXIMUM_ERROR_POINT");
|
||||
|
||||
return outputs;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration config) throws Exception {
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
// init db connection
|
||||
connection = AlgorithmConfiguration.getConnectionFromConfig(config);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HashMap<String, String> analyze(AlgorithmConfiguration config) throws Exception {
|
||||
|
||||
String FirstTableCsquareColumn = config.getParam("FirstTableCsquareColumn");
|
||||
String SecondTableCsquareColumn = config.getParam("SecondTableCsquareColumn");
|
||||
String FirstTableProbabilityColumn = config.getParam("FirstTableProbabilityColumn");
|
||||
String SecondTableProbabilityColumn = config.getParam("SecondTableProbabilityColumn");
|
||||
String FirstTable = config.getParam("FirstTable");
|
||||
String SecondTable = config.getParam("SecondTable");
|
||||
|
||||
String query = String.format(discrepancyQuery, FirstTableCsquareColumn, SecondTableCsquareColumn, FirstTableProbabilityColumn, SecondTableProbabilityColumn, FirstTable, SecondTable);
|
||||
|
||||
List<Object> takeNPoints = DatabaseFactory.executeSQLQuery(String.format(getNumberOfElementsQuery, FirstTable), connection);
|
||||
List<Object> takeMPoints = DatabaseFactory.executeSQLQuery(String.format(getNumberOfElementsQuery, SecondTable), connection);
|
||||
int nPoints = Integer.parseInt(""+takeNPoints.get(0));
|
||||
int mPoints = Integer.parseInt(""+takeMPoints.get(0));
|
||||
numberofvectors = Math.max(nPoints, mPoints);
|
||||
|
||||
AnalysisLogger.getLogger().trace("Discrepancy Calculation - Query to perform :" + query);
|
||||
List<Object> takePoints = DatabaseFactory.executeSQLQuery(query, connection);
|
||||
|
||||
super.processedRecords = 0;
|
||||
if (takePoints != null)
|
||||
super.processedRecords = takePoints.size();
|
||||
|
||||
threshold = Float.parseFloat(config.getParam("ComparisonThreshold"));
|
||||
analyzeCompareList(takePoints);
|
||||
calcDiscrepancy();
|
||||
|
||||
HashMap<String, String> output = new HashMap<String, String>();
|
||||
output.put("MEAN", "" + mean);
|
||||
output.put("VARIANCE", "" + variance);
|
||||
output.put("NUMBER_OF_ERRORS", "" + numberoferrors);
|
||||
output.put("NUMBER_OF_COMPARISONS", "" + numberofvectors);
|
||||
|
||||
float accuracy = 100;
|
||||
if (processedRecords>0)
|
||||
accuracy = (1 - (float) numberoferrors / (float) numberofvectors) * 100;
|
||||
|
||||
|
||||
output.put("ACCURACY", "" + accuracy);
|
||||
output.put("MAXIMUM_ERROR", "" + maxerror);
|
||||
output.put("MAXIMUM_ERROR_POINT", "" + maxdiscrepancyPoint);
|
||||
|
||||
return output;
|
||||
|
||||
}
|
||||
|
||||
public void end() {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
void calcDiscrepancy() {
|
||||
double[] err = new double[errors.size()];
|
||||
int i = 0;
|
||||
for (Float e : errors) {
|
||||
err[i] = e;
|
||||
i++;
|
||||
}
|
||||
|
||||
mean = 0;
|
||||
variance = 0;
|
||||
|
||||
if (err.length > 0) {
|
||||
mean = MathFunctions.mean(err);
|
||||
variance = com.rapidminer.tools.math.MathFunctions.variance(err, Double.NEGATIVE_INFINITY);
|
||||
}
|
||||
}
|
||||
|
||||
public void analyzeCompareList(List<Object> points) {
|
||||
errors = new ArrayList<Float>();
|
||||
|
||||
if (points != null) {
|
||||
maxerror = 0;
|
||||
for (Object vector : points) {
|
||||
Object[] elements = (Object[]) vector;
|
||||
String csquare = (String) elements[0];
|
||||
float probabilityPoint1 = 0;
|
||||
if (elements[2] != null)
|
||||
probabilityPoint1 = (Float) elements[2];
|
||||
float probabilityPoint2 = 0;
|
||||
if (elements[3] != null)
|
||||
probabilityPoint2 = (Float) elements[3];
|
||||
float discrepancy = Math.abs(probabilityPoint2 - probabilityPoint1);
|
||||
|
||||
if (discrepancy > threshold) {
|
||||
errors.add(Math.abs(probabilityPoint2 - probabilityPoint1));
|
||||
numberoferrors++;
|
||||
if (discrepancy > maxerror) {
|
||||
maxerror = discrepancy;
|
||||
maxdiscrepancyPoint = csquare;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,324 @@
|
|||
package org.gcube.dataanalysis.ecoengine.evaluation;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.DataAnalysis;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
import com.rapidminer.example.Attribute;
|
||||
import com.rapidminer.example.Attributes;
|
||||
import com.rapidminer.example.ExampleSet;
|
||||
import com.rapidminer.example.table.AttributeFactory;
|
||||
import com.rapidminer.example.table.BinominalMapping;
|
||||
import com.rapidminer.example.table.DoubleArrayDataRow;
|
||||
import com.rapidminer.example.table.MemoryExampleTable;
|
||||
import com.rapidminer.tools.Ontology;
|
||||
import com.rapidminer.tools.math.ROCData;
|
||||
import com.rapidminer.tools.math.ROCDataGenerator;
|
||||
|
||||
public class DistributionQualityAnalysis extends DataAnalysis {
|
||||
|
||||
static String getProbabilititesQuery = "select count(*) as distribprob from %1$s as a join %2$s as b on a.%3$s=b.%4$s and b.%5$s %6$s %7$s";
|
||||
static String getNumberOfElementsQuery = "select count(*) from %1$s";
|
||||
|
||||
static String getValuesQuery = "select %5$s as distribprob from %1$s as a join %2$s as b on a.%3$s=b.%4$s";
|
||||
|
||||
float threshold = 0.1f;
|
||||
String configPath = "./cfg/";
|
||||
float acceptanceThreshold = 0.8f;
|
||||
float rejectionThreshold = 0.3f;
|
||||
double bestThreshold = 0.5d;
|
||||
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
|
||||
parameters.put("PositiveCasesTable", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("NegativeCasesTable", new VarCouple(VARTYPE.STRING, ""));
|
||||
|
||||
parameters.put("PositiveCasesTableKeyColumn", new VarCouple(VARTYPE.STRING, "csquarecode"));
|
||||
parameters.put("NegativeCasesTableKeyColumn", new VarCouple(VARTYPE.STRING, "csquarecode"));
|
||||
|
||||
parameters.put("DistributionTable", new VarCouple(VARTYPE.STRING, "csquare"));
|
||||
parameters.put("DistributionTableKeyColumn", new VarCouple(VARTYPE.STRING, "csquarecode"));
|
||||
parameters.put("DistributionTableProbabilityColumn", new VarCouple(VARTYPE.STRING, "csquarecode"));
|
||||
|
||||
parameters.put("PositiveThreshold", new VarCouple(VARTYPE.STRING, "0.8"));
|
||||
parameters.put("NegativeThreshold", new VarCouple(VARTYPE.STRING, "0.3"));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
public List<String> getOutputParameters() {
|
||||
|
||||
List<String> outputs = new ArrayList<String>();
|
||||
|
||||
outputs.add("TRUE_POSITIVES");
|
||||
outputs.add("TRUE_NEGATIVES");
|
||||
outputs.add("FALSE_POSITIVES");
|
||||
outputs.add("FALSE_NEGATIVES");
|
||||
outputs.add("AUC");
|
||||
outputs.add("ACCURACY");
|
||||
outputs.add("SENSITIVITY");
|
||||
outputs.add("OMISSIONRATE");
|
||||
outputs.add("SPECIFICITY");
|
||||
outputs.add("BESTTHRESHOLD");
|
||||
|
||||
return outputs;
|
||||
}
|
||||
|
||||
private int calculateNumberOfPoints(String table) {
|
||||
|
||||
String numberOfPositiveCasesQuery = String.format(getNumberOfElementsQuery, table);
|
||||
List<Object> totalPoints = DatabaseFactory.executeSQLQuery(numberOfPositiveCasesQuery, connection);
|
||||
int points = Integer.parseInt("" + totalPoints.get(0));
|
||||
return points;
|
||||
}
|
||||
|
||||
private int calculateCaughtPoints(String casesTable, String distributionTable, String casesTableKeyColumn, String distributionTableKeyColumn, String distributionTableProbabilityColumn, String operator, String threshold) {
|
||||
|
||||
String query = String.format(getProbabilititesQuery, casesTable, distributionTable, casesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, operator, threshold);
|
||||
AnalysisLogger.getLogger().trace("Compare - Query to perform for caught cases:" + query);
|
||||
List<Object> caughtpoints = DatabaseFactory.executeSQLQuery(query, connection);
|
||||
int points = Integer.parseInt("" + caughtpoints.get(0));
|
||||
return points;
|
||||
}
|
||||
|
||||
private double[] getPoints(String casesTable, String distributionTable, String casesTableKeyColumn, String distributionTableKeyColumn, String distributionTableProbabilityColumn, int numberOfExpectedPoints) {
|
||||
|
||||
String query = String.format(getValuesQuery, casesTable, distributionTable, casesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn);
|
||||
|
||||
AnalysisLogger.getLogger().trace("Compare - Query to perform for caught cases:" + query);
|
||||
List<Object> caughtpoints = DatabaseFactory.executeSQLQuery(query, connection);
|
||||
int size = 0;
|
||||
if (caughtpoints != null)
|
||||
size = caughtpoints.size();
|
||||
double[] points = new double[numberOfExpectedPoints];
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
double element = 0;
|
||||
if (caughtpoints.get(i) != null)
|
||||
element = Double.parseDouble("" + caughtpoints.get(i));
|
||||
|
||||
points[i] = element;
|
||||
}
|
||||
|
||||
return points;
|
||||
}
|
||||
|
||||
public HashMap<String, String> analyze(AlgorithmConfiguration config) throws Exception {
|
||||
|
||||
try {
|
||||
acceptanceThreshold = Float.parseFloat(config.getParam("PositiveThreshold"));
|
||||
} catch (Exception e) {
|
||||
AnalysisLogger.getLogger().debug("ERROR : " + e.getLocalizedMessage());
|
||||
}
|
||||
try {
|
||||
rejectionThreshold = Float.parseFloat(config.getParam("NegativeThreshold"));
|
||||
} catch (Exception e) {
|
||||
AnalysisLogger.getLogger().debug("ERROR : " + e.getLocalizedMessage());
|
||||
}
|
||||
|
||||
String positiveCasesTable = config.getParam("PositiveCasesTable");
|
||||
String negativeCasesTable = config.getParam("NegativeCasesTable");
|
||||
String distributionTable = config.getParam("DistributionTable");
|
||||
String positiveCasesTableKeyColumn = config.getParam("PositiveCasesTableKeyColumn");
|
||||
String negativeCasesTableKeyColumn = config.getParam("NegativeCasesTableKeyColumn");
|
||||
String distributionTableKeyColumn = config.getParam("DistributionTableKeyColumn");
|
||||
String distributionTableProbabilityColumn = config.getParam("DistributionTableProbabilityColumn");
|
||||
String acceptanceThreshold = config.getParam("PositiveThreshold");
|
||||
String rejectionThreshold = config.getParam("NegativeThreshold");
|
||||
|
||||
int numberOfPositiveCases = calculateNumberOfPoints(config.getParam("PositiveCasesTable"));
|
||||
|
||||
int truePositives = calculateCaughtPoints(positiveCasesTable, distributionTable, positiveCasesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, ">", acceptanceThreshold);
|
||||
|
||||
int falseNegatives = numberOfPositiveCases - truePositives;
|
||||
|
||||
int numberOfNegativeCases = calculateNumberOfPoints(negativeCasesTable);
|
||||
|
||||
super.processedRecords = numberOfPositiveCases + numberOfNegativeCases;
|
||||
|
||||
int falsePositives = calculateCaughtPoints(negativeCasesTable, distributionTable, negativeCasesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, ">", rejectionThreshold);
|
||||
|
||||
int trueNegatives = numberOfNegativeCases - falsePositives;
|
||||
|
||||
double[] positivePoints = getPoints(positiveCasesTable, distributionTable, positiveCasesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, numberOfPositiveCases);
|
||||
|
||||
double[] negativePoints = getPoints(negativeCasesTable, distributionTable, negativeCasesTableKeyColumn, distributionTableKeyColumn, distributionTableProbabilityColumn, numberOfNegativeCases);
|
||||
|
||||
double auc = calculateAUC(positivePoints, negativePoints, false);
|
||||
double accuracy = calculateAccuracy(truePositives, trueNegatives, falsePositives, falseNegatives);
|
||||
double sensitivity = calculateSensitivity(truePositives, falseNegatives);
|
||||
double omissionrate = calculateOmissionRate(truePositives, falseNegatives);
|
||||
double specificity = calculateSpecificity(trueNegatives, falsePositives);
|
||||
|
||||
HashMap<String, String> output = new HashMap<String, String>();
|
||||
output.put("TRUE_POSITIVES", "" + truePositives);
|
||||
output.put("TRUE_NEGATIVES", "" + trueNegatives);
|
||||
output.put("FALSE_POSITIVES", "" + falsePositives);
|
||||
output.put("FALSE_NEGATIVES", "" + falseNegatives);
|
||||
output.put("AUC", "" + auc);
|
||||
output.put("ACCURACY", "" + accuracy);
|
||||
output.put("SENSITIVITY", "" + sensitivity);
|
||||
output.put("OMISSIONRATE", "" + omissionrate);
|
||||
output.put("SPECIFICITY", "" + specificity);
|
||||
output.put("BESTTHRESHOLD", "" + bestThreshold);
|
||||
|
||||
return output;
|
||||
|
||||
}
|
||||
|
||||
public double calculateSensitivity(int TP, int FN) {
|
||||
return (double) (TP) / (double) (TP + FN);
|
||||
}
|
||||
|
||||
public double calculateOmissionRate(int TP, int FN) {
|
||||
return (double) (FN) / (double) (TP + FN);
|
||||
}
|
||||
|
||||
public double calculateSpecificity(int TN, int FP) {
|
||||
return (double) (TN) / (double) (TN + FP);
|
||||
}
|
||||
|
||||
public double calculateAccuracy(int TP, int TN, int FP, int FN) {
|
||||
return (double) (TP + TN) / (double) (TP + TN + FP + FN);
|
||||
}
|
||||
|
||||
public double calculateAUC(double[] scoresOnPresence, double[] scoresOnAbsence, boolean produceChart) {
|
||||
|
||||
List<Attribute> attributes = new LinkedList<Attribute>();
|
||||
Attribute labelAtt = AttributeFactory.createAttribute("LABEL", Ontology.BINOMINAL);
|
||||
BinominalMapping bm = (BinominalMapping) labelAtt.getMapping();
|
||||
bm.setMapping("1", 1);
|
||||
bm.setMapping("0", 0);
|
||||
|
||||
Attribute confidenceAtt1 = AttributeFactory.createAttribute(Attributes.CONFIDENCE_NAME + "_1", Ontology.REAL);
|
||||
attributes.add(confidenceAtt1);
|
||||
attributes.add(labelAtt);
|
||||
|
||||
MemoryExampleTable table = new MemoryExampleTable(attributes);
|
||||
int numOfPoints = scoresOnPresence.length + scoresOnAbsence.length;
|
||||
int numOfPresence = scoresOnPresence.length;
|
||||
int numOfAttributes = attributes.size();
|
||||
double pos = labelAtt.getMapping().mapString("1");
|
||||
double neg = labelAtt.getMapping().mapString("0");
|
||||
|
||||
for (int i = 0; i < numOfPresence; i++) {
|
||||
double[] data = new double[numOfAttributes];
|
||||
data[0] = scoresOnPresence[i];
|
||||
data[1] = pos;
|
||||
table.addDataRow(new DoubleArrayDataRow(data));
|
||||
}
|
||||
|
||||
for (int i = numOfPresence; i < numOfPoints; i++) {
|
||||
double[] data = new double[numOfAttributes];
|
||||
data[0] = scoresOnAbsence[i - numOfPresence];
|
||||
data[1] = neg;
|
||||
table.addDataRow(new DoubleArrayDataRow(data));
|
||||
}
|
||||
|
||||
ROCDataGenerator roc = new ROCDataGenerator(acceptanceThreshold, rejectionThreshold);
|
||||
ExampleSet exampleSet = table.createExampleSet(labelAtt);
|
||||
exampleSet.getAttributes().setSpecialAttribute(confidenceAtt1, Attributes.CONFIDENCE_NAME + "_1");
|
||||
|
||||
ROCData dataROC = roc.createROCData(exampleSet, false);
|
||||
double auc = roc.calculateAUC(dataROC);
|
||||
|
||||
// PLOTS THE ROC!!!
|
||||
if (produceChart)
|
||||
roc.createROCPlotDialog(dataROC);
|
||||
|
||||
bestThreshold = roc.getBestThreshold();
|
||||
return auc;
|
||||
}
|
||||
|
||||
public static void visualizeResults(HashMap<String, String> results) {
|
||||
|
||||
for (String key : results.keySet()) {
|
||||
System.out.println(key + ":" + results.get(key));
|
||||
}
|
||||
}
|
||||
|
||||
SessionFactory connection;
|
||||
|
||||
public void init(AlgorithmConfiguration config) throws Exception {
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
// init db connection
|
||||
connection = AlgorithmConfiguration.getConnectionFromConfig(config);
|
||||
}
|
||||
|
||||
public void end() {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile);
|
||||
/*
|
||||
* double [] pos = new double [4]; // pos[0] = 1d; pos[1] = 0.8d;pos[2]=0.7;pos[3]=0.9; // pos[0] = 1d; pos[1] = 1d;pos[2]=1;pos[3]=1; // pos[0] = 0.3d; pos[1] = 0.7d;pos[2]=0.1;pos[3]=0.9;
|
||||
*
|
||||
*
|
||||
* double [] neg = new double [4]; // neg[0] = 0d; neg[1] = 0.3d;neg[2]=0.4;neg[3]=0.6; // neg[0] = 0d; neg[1] = 0.0d;neg[2]=0.0;neg[3]=0.0; // neg[0] = 0.7d; neg[1] = 0.3d;neg[2]=0.9;neg[3]=0.1;
|
||||
*
|
||||
* DistributionQualityAnalysis quality = new DistributionQualityAnalysis(); double auc = quality.calculateAUC(pos, neg); System.out.println("AUC: "+auc);
|
||||
*
|
||||
* int n = 100; double[] posRandom = new double[n]; double[] negRandom = new double[n];
|
||||
*
|
||||
* for (int i=0;i<n;i++){ posRandom[i] = Math.random(); negRandom[i] = Math.random(); }
|
||||
*
|
||||
*
|
||||
* quality = new DistributionQualityAnalysis(); auc = quality.calculateAUC(posRandom, negRandom); System.out.println("AUC: "+auc);
|
||||
*
|
||||
* for (int i=0;i<n;i++){ posRandom[i] = 1; negRandom[i] = 0; }
|
||||
*
|
||||
* quality = new DistributionQualityAnalysis(); auc = quality.calculateAUC(posRandom, negRandom); System.out.println("AUC: "+auc);
|
||||
*/
|
||||
|
||||
int n = 100;
|
||||
double[] posRandom = new double[n];
|
||||
double[] negRandom = new double[n];
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
posRandom[i] = Math.random();
|
||||
negRandom[i] = Math.random();
|
||||
}
|
||||
|
||||
DistributionQualityAnalysis quality = new DistributionQualityAnalysis();
|
||||
double auc = quality.calculateAUC(posRandom, negRandom, true);
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
posRandom[i] = 1;
|
||||
negRandom[i] = 0;
|
||||
}
|
||||
|
||||
quality = new DistributionQualityAnalysis();
|
||||
auc = quality.calculateAUC(posRandom, negRandom, true);
|
||||
System.out.println("AUC: " + auc);
|
||||
|
||||
double[] po = { 0.16, 0.12, 0.12, 0.16, 0.58, 0.36, 0.32, 0.5, 0.65, 0.59, 0.65, 0.65, 0.65, 0.38, 0.18, 0.64, 0.28, 0.64, 0.52, 0.72, 0.74, 0.23, 0.23, 0.23, 0.21, 0.21, 0.22, 0.22, 0.24, 0.32, 0.32, 0.32, 0.32, 0.55, 0.78, 0.37, 0.87, 0.87, 0.87, 0.98, 0.98, 0.76, 0.76, 0.9, 0.88, 0.97, 0.97, 0.97, 1.0, 1.0, 0.45, 0.45, 0.19, 0.89, 0.17, 0.16, 0.1, 0.25, 0.89, 0.89, 0.9, 0.9, 0.87, 1.0, 0.48, 0.88, 0.9, 0.93, 1.0, 1.0, 0.17, 0.87, 1.0, 0.24, 0.86, 0.15, 0.74, 0.32, 1.0, 0.95, 0.52, 0.66, 0.39, 0.31, 0.47, 0.57, 0.73, 0.83, 0.86, 0.98, 0.99, 1.0, 1.0, 1.0, 1.0, 0.86, 0.43, 0.67, 0.66, 0.41, 0.52, 0.46, 0.34, 1.0, 1.0, 1.0, 0.68, 1.0, 0.98, 0.89, 0.79, 1.0, 0.88, 0.99, 1.0, 0.95, 0.95, 0.95, 0.95, 0.88, 0.96, 0.95, 0.96, 0.99, 1.0, 0.98, 0.6, 0.36, 0.15, 0.87, 0.43, 0.86, 0.34, 0.21, 0.41, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.94, 0.98, 0.71, 0.85, 0.49, 0.91, 0.75, 0.74, 0.42, 0.99, 0.43, 0.22, 0.23, 1.0, 1.0, 1.0, 1.0, 0.4, 1.0, 1.0, 1.0, 0.94, 0.95, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.94, 0.98, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.97, 0.11, 0.12, 0.19, 0.2, 0.46, 0.9, 0.84, 0.64, 1.0, 0.77, 0.56, 0.66, 0.17, 0.54, 0.2, 0.27, 0.24, 0.52, 0.74, 0.23, 0.78, 0.69, 0.46, 0.65, 0.18, 0.28, 0.66, 0.66, 0.6, 0.16, 0.24, 0.4, 0.79, 0.69, 0.81, 0.49, 0.29, 0.5, 0.46, 0.15, 0.29, 0.54, 0.29, 0.37, 0.12, 0.24, 0.16, 0.4, 0.24, 0.55, 0.68, 0.6, 0.14, 0.56, 0.17, 0.73, 0.73, 0.43, 0.72, 0.72, 0.49, 0.13, 0.37, 0.11, 0.25, 0.11, 0.74, 0.59, 0.35, 0.67, 0.83, 0.71, 0.48, 0.86, 0.94, 0.17, 0.19, 0.13, 0.27, 0.77, 0.38, 0.47, 0.49, 0.13, 0.27, 0.14, 0.4, 0.45, 0.15, 0.68, 0.37, 0.2, 0.2, 0.63, 0.35, 0.13, 0.17, 0.24, 0.85, 0.58, 0.44, 1.0, 1.0, 0.94, 0.58, 0.28, 0.36, 0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
|
||||
double[] ne = { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 };
|
||||
|
||||
quality = new DistributionQualityAnalysis();
|
||||
auc = quality.calculateAUC(po, ne, true);
|
||||
System.out.println("AUC: " + auc);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
package org.gcube.dataanalysis.ecoengine.interfaces;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.WEIGHT;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
|
||||
/**
|
||||
* Implements a mono-thread data analysis process
|
||||
* Status is managed outside the class and analysis is simply conducted by initializing and applying procedure
|
||||
* @author coro
|
||||
*
|
||||
*/
|
||||
public abstract class DataAnalysis implements Evaluator{
|
||||
|
||||
protected ResourceFactory resourceManager;
|
||||
protected int processedRecords;
|
||||
protected float status;
|
||||
|
||||
/**
|
||||
* establishes imput parameters for this algorithm along with their type
|
||||
*/
|
||||
public abstract HashMap<String, VarCouple> getInputParameters();
|
||||
|
||||
/**
|
||||
* lists the output parameters names
|
||||
* @return
|
||||
*/
|
||||
public abstract List<String> getOutputParameters();
|
||||
|
||||
/**
|
||||
* Executed the core of the algorithm
|
||||
* @param config
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
public abstract HashMap<String, String> analyze(AlgorithmConfiguration config) throws Exception;
|
||||
|
||||
/**
|
||||
* initializes the procedure e.g. connects to the database
|
||||
* @param config
|
||||
* @throws Exception
|
||||
*/
|
||||
public abstract void init(AlgorithmConfiguration config) throws Exception;
|
||||
|
||||
/**
|
||||
* ends the processing, e.g. closes connections
|
||||
* @throws Exception
|
||||
*/
|
||||
public abstract void end();
|
||||
|
||||
/**
|
||||
* Processing skeleton : init-analyze-end
|
||||
* @param config
|
||||
* @return
|
||||
* @throws Exception
|
||||
*/
|
||||
public HashMap<String, String> process(AlgorithmConfiguration config) throws Exception{
|
||||
status = 0;
|
||||
HashMap<String, String> out = new HashMap<String, String>();
|
||||
try{
|
||||
init(config);
|
||||
out = analyze(config);
|
||||
end();
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
throw e;
|
||||
}
|
||||
finally{
|
||||
status = 100;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* calculates the number of processed records per unity of time: the timing is calculated internally by the resourceManager and used when the method is interrogated
|
||||
*/
|
||||
@Override
|
||||
public String getResourceLoad() {
|
||||
if (resourceManager==null)
|
||||
resourceManager = new ResourceFactory();
|
||||
return resourceManager.getResourceLoad(processedRecords);
|
||||
}
|
||||
|
||||
/**
|
||||
* gets the occupancy of the resource: in this case one thread
|
||||
*/
|
||||
@Override
|
||||
public String getResources() {
|
||||
return ResourceFactory.getResources(100f);
|
||||
}
|
||||
|
||||
/**
|
||||
* The weight of this procedure is the lowest as it runs on local machine
|
||||
*/
|
||||
@Override
|
||||
public WEIGHT getWeight() {
|
||||
return WEIGHT.LOWEST;
|
||||
}
|
||||
|
||||
/**
|
||||
* gets the internal status of the operation
|
||||
*/
|
||||
@Override
|
||||
public float getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* visualizes the results of the analysis
|
||||
* @param results
|
||||
*/
|
||||
public static void visualizeResults(HashMap<String,String> results){
|
||||
|
||||
for (String key:results.keySet()){
|
||||
System.out.println(key+":"+results.get(key));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package org.gcube.dataanalysis.ecoengine.interfaces;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
|
||||
|
||||
public interface Evaluator extends ComputationalAgent{
|
||||
|
||||
|
||||
public HashMap<String, String> process(AlgorithmConfiguration config) throws Exception;
|
||||
|
||||
public abstract void init(AlgorithmConfiguration config) throws Exception;
|
||||
|
||||
public abstract void end();
|
||||
|
||||
}
|
||||
|
|
@ -54,7 +54,7 @@ public interface SpatialProbabilityDistributionTable extends SpatialProbabilityD
|
|||
//apply a filter to a single table row representing a probability point
|
||||
public String filterProbabiltyRow(String probabiltyRow);
|
||||
|
||||
//apply a bulk filter
|
||||
//apply a bulk filter when a synchronous write is enabled
|
||||
public Queue<String> filterProbabilitySet(Queue<String> probabiltyRows);
|
||||
|
||||
//indicate if the write of the probability rows will be during the overall computation for a single mainInformation object or after the whole processing
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ public class SimpleModeler implements Modeler{
|
|||
|
||||
@Override
|
||||
public ALG_PROPS[] getSupportedModels() {
|
||||
ALG_PROPS[] props = {ALG_PROPS.SPECIES_ENVELOPES};
|
||||
ALG_PROPS[] props = {ALG_PROPS.SPECIES_ENVELOPES,ALG_PROPS.SPECIES_MODEL};
|
||||
return props;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,222 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Model;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class ModelAquamapsNN implements Model {
|
||||
|
||||
@Override
|
||||
public ALG_PROPS[] getProperties() {
|
||||
ALG_PROPS[] props = { ALG_PROPS.SPECIES_MODEL };
|
||||
return props;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "AQUAMAPSNN";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Aquamaps Trained using Neural Networks";
|
||||
}
|
||||
|
||||
@Override
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
parameters.put("AbsenceDataTable", new VarCouple(VARTYPE.STRING, "absence_data"));
|
||||
parameters.put("PresenceDataTable", new VarCouple(VARTYPE.STRING, "presence_data"));
|
||||
parameters.put("SpeciesName", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("UserName", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("LayersNeurons", new VarCouple(VARTYPE.STRING, "100,2"));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getVersion() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setVersion(float version) {
|
||||
}
|
||||
|
||||
SessionFactory connection;
|
||||
String fileName;
|
||||
String presenceTable;
|
||||
String absenceTable;
|
||||
float status;
|
||||
int[] layersNeurons = {100, 2};
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration Input, Model previousModel) {
|
||||
AnalysisLogger.setLogger(Input.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
// init the database
|
||||
String defaultDatabaseFile = Input.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile;
|
||||
|
||||
Input.setDatabaseDriver(Input.getParam("DatabaseDriver"));
|
||||
Input.setDatabaseUserName(Input.getParam("DatabaseUserName"));
|
||||
Input.setDatabasePassword(Input.getParam("DatabasePassword"));
|
||||
Input.setDatabaseURL(Input.getParam("DatabaseURL"));
|
||||
|
||||
try {
|
||||
connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, Input);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().trace("ERROR initializing connection");
|
||||
}
|
||||
|
||||
fileName = Input.getPersistencePath() + "neuralnetwork_" + Input.getParam("SpeciesName") + "_" + Input.getParam("UserName");
|
||||
presenceTable = Input.getParam("PresenceDataTable");
|
||||
absenceTable = Input.getParam("AbsenceDataTable");
|
||||
|
||||
String layersNeurons$ = Input.getParam("LayersNeurons");
|
||||
if ((layersNeurons$!=null)&&(layersNeurons$.length()>0))
|
||||
{
|
||||
String [] split = layersNeurons$.split(",");
|
||||
layersNeurons = new int[split.length];
|
||||
for (int i = 0;i<split.length;i++){
|
||||
layersNeurons[i] = Integer.parseInt(split[i]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getResourceLoad() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getResources() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getInputType() {
|
||||
return AlgorithmConfiguration.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOutputType() {
|
||||
return String.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void postprocess(AlgorithmConfiguration Input, Model previousModel) {
|
||||
connection.close();
|
||||
}
|
||||
|
||||
// private String takeElementsQuery = "select depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea,centerlat,centerlong from %1$s d where oceanarea>0 limit 100";
|
||||
private String takeElementsQuery = "select depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea from %1$s d where oceanarea>0 limit 449";
|
||||
// private String takeAElementsQuery = "select depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea from %1$s d where oceanarea>0 limit 100";
|
||||
|
||||
@Override
|
||||
public void train(AlgorithmConfiguration Input, Model previousModel) {
|
||||
|
||||
try {
|
||||
// take all presence inputs
|
||||
List<Object> presences = DatabaseFactory.executeSQLQuery(String.format(takeElementsQuery, presenceTable), connection);
|
||||
// take all absence inputs
|
||||
// AnalysisLogger.getLogger().trace("presence "+String.format(takeElementsQuery, presenceTable));
|
||||
// AnalysisLogger.getLogger().trace("absence "+String.format(takeElementsQuery, absenceTable));
|
||||
List<Object> absences = DatabaseFactory.executeSQLQuery(String.format(takeElementsQuery, absenceTable), connection);
|
||||
int numbOfPresence = presences.size();
|
||||
int numbOfAbsence = absences.size();
|
||||
|
||||
// setup Neural Network
|
||||
int numberOfInputNodes = 11;
|
||||
int numberOfOutputNodes = 1;
|
||||
int[] innerLayers = Neural_Network.setupInnerLayers(layersNeurons);
|
||||
Neural_Network nn = new Neural_Network(numberOfInputNodes, numberOfOutputNodes, innerLayers, Neural_Network.ACTIVATIONFUNCTION.SIGMOID);
|
||||
|
||||
|
||||
int numberOfInputs = numbOfPresence + numbOfAbsence;
|
||||
double[][] in = new double[numberOfInputs][];
|
||||
double[][] out = new double[numberOfInputs][];
|
||||
// build NN input
|
||||
for (int i = 0; i < numbOfPresence; i++) {
|
||||
in[i] = Neural_Network.preprocessObjects((Object[]) presences.get(i));
|
||||
out[i] = nn.getPositiveCase();
|
||||
}
|
||||
for (int i = numbOfPresence; i < numberOfInputs; i++) {
|
||||
in[i] = Neural_Network.preprocessObjects((Object[]) absences.get(i-numbOfPresence));
|
||||
out[i] = nn.getNegativeCase();
|
||||
}
|
||||
|
||||
// train the NN
|
||||
nn.train(in, out);
|
||||
save(fileName, nn);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().error("ERROR during training");
|
||||
}
|
||||
status = 100f;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
return String.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getContent() {
|
||||
return fileName;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void save(String nomeFile, Neural_Network nn) {
|
||||
|
||||
File f = new File(nomeFile);
|
||||
FileOutputStream stream = null;
|
||||
try {
|
||||
stream = new FileOutputStream(f);
|
||||
ObjectOutputStream oos = new ObjectOutputStream(stream);
|
||||
oos.writeObject(nn);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().error("ERROR in writing object on file: " + nomeFile);
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
AnalysisLogger.getLogger().trace("OK in writing object on file: " + nomeFile);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,227 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Model;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions.NeuralNet;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions.Pattern;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class ModelAquamapsNNNS implements Model {
|
||||
|
||||
@Override
|
||||
public ALG_PROPS[] getProperties() {
|
||||
ALG_PROPS[] props = { ALG_PROPS.SPECIES_MODEL };
|
||||
return props;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "AQUAMAPSNNNS";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Aquamaps Trained using Neural Networks";
|
||||
}
|
||||
|
||||
@Override
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
parameters.put("AbsenceDataTable", new VarCouple(VARTYPE.STRING, "absence_data"));
|
||||
parameters.put("PresenceDataTable", new VarCouple(VARTYPE.STRING, "presence_data"));
|
||||
parameters.put("SpeciesName", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("UserName", new VarCouple(VARTYPE.STRING, ""));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getVersion() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setVersion(float version) {
|
||||
}
|
||||
|
||||
SessionFactory connection;
|
||||
String fileName;
|
||||
String presenceTable;
|
||||
String absenceTable;
|
||||
float status;
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration Input, Model previousModel) {
|
||||
AnalysisLogger.setLogger(Input.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
// init the database
|
||||
String defaultDatabaseFile = Input.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile;
|
||||
|
||||
Input.setDatabaseDriver(Input.getParam("DatabaseDriver"));
|
||||
Input.setDatabaseUserName(Input.getParam("DatabaseUserName"));
|
||||
Input.setDatabasePassword(Input.getParam("DatabasePassword"));
|
||||
Input.setDatabaseURL(Input.getParam("DatabaseURL"));
|
||||
|
||||
try {
|
||||
connection = DatabaseFactory.initDBConnection(defaultDatabaseFile, Input);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().trace("ERROR initializing connection");
|
||||
}
|
||||
|
||||
fileName = Input.getPersistencePath() + "neuralnetwork_" + Input.getParam("SpeciesName") + "_" + Input.getParam("UserName");
|
||||
presenceTable = Input.getParam("PresenceDataTable");
|
||||
absenceTable = Input.getParam("AbsenceDataTable");
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getResourceLoad() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getResources() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getInputType() {
|
||||
return AlgorithmConfiguration.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOutputType() {
|
||||
return String.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void postprocess(AlgorithmConfiguration Input, Model previousModel) {
|
||||
connection.close();
|
||||
}
|
||||
|
||||
private String takeElementsQuery = "select depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea from %1$s d where oceanarea>0 limit 449";
|
||||
|
||||
@Override
|
||||
public void train(AlgorithmConfiguration Input, Model previousModel) {
|
||||
|
||||
try {
|
||||
// take all presence inputs
|
||||
List<Object> presences = DatabaseFactory.executeSQLQuery(String.format(takeElementsQuery, presenceTable), connection);
|
||||
// take all absence inputs
|
||||
List<Object> absences = DatabaseFactory.executeSQLQuery(String.format(takeElementsQuery, absenceTable), connection);
|
||||
int numbOfPresence = presences.size();
|
||||
int numbOfAbsence = absences.size();
|
||||
|
||||
// setup Neural Network
|
||||
int numberOfInputNodes = 11;
|
||||
int numberOfOutputNodes = 1;
|
||||
// int[] innerLayers = Neural_Network.setupInnerLayers(100,30,10);
|
||||
// int[] innerLayers = NeuralNet.setupInnerLayers(100,10,30);
|
||||
int[] innerLayers = NeuralNet.setupInnerLayers(140);
|
||||
NeuralNet nn = new NeuralNet(numberOfInputNodes, numberOfOutputNodes, innerLayers);
|
||||
|
||||
|
||||
int numberOfInputs = numbOfPresence + numbOfAbsence;
|
||||
double[][] in = new double[numberOfInputs][];
|
||||
double[][] out = new double[numberOfInputs][];
|
||||
// build NN input
|
||||
for (int i = 0; i < numbOfPresence; i++) {
|
||||
in[i] = NeuralNet.preprocessObjects((Object[]) presences.get(i));
|
||||
out[i] = nn.getPositiveCase();
|
||||
Pattern pattern = new Pattern(in[i], out[i]);
|
||||
nn.IncrementalTrain(.2, pattern);
|
||||
AnalysisLogger.getLogger().debug("-> "+i);
|
||||
}
|
||||
for (int i = numbOfPresence; i < numberOfInputs; i++) {
|
||||
in[i] = NeuralNet.preprocessObjects((Object[]) absences.get(i-numbOfPresence));
|
||||
out[i] = nn.getNegativeCase();
|
||||
Pattern pattern = new Pattern(in[i], out[i]);
|
||||
nn.IncrementalTrain(.2, pattern);
|
||||
AnalysisLogger.getLogger().debug("-> "+i);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
int numberOfInputs = numbOfPresence;
|
||||
double[][] in = new double[numberOfInputs][];
|
||||
double[][] out = new double[numberOfInputs][];
|
||||
// build NN input
|
||||
for (int i = 0; i < numbOfPresence; i++) {
|
||||
in[i] = Neural_Network.preprocessObjects((Object[]) presences.get(i));
|
||||
out[i] = nn.getPositiveCase();
|
||||
}
|
||||
*/
|
||||
|
||||
// train the NN
|
||||
save(fileName, nn);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().error("ERROR during training");
|
||||
}
|
||||
status = 100f;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
return String.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getContent() {
|
||||
return fileName;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void save(String nomeFile, NeuralNet nn) {
|
||||
|
||||
File f = new File(nomeFile);
|
||||
FileOutputStream stream = null;
|
||||
try {
|
||||
stream = new FileOutputStream(f);
|
||||
ObjectOutputStream oos = new ObjectOutputStream(stream);
|
||||
oos.writeObject(nn);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
AnalysisLogger.getLogger().error("ERROR in writing object on file: " + nomeFile);
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
AnalysisLogger.getLogger().trace("OK in writing object on file: " + nomeFile);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,570 @@
|
|||
//Classe neural_network:le sue istanze sono reti neurali feed forward multistato
|
||||
//implementate come una "griglia" di istanze della classe Neuron.E' possibile scegliere
|
||||
//il numero di layer e quello dei neuroni per ognuno di questi.
|
||||
|
||||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class Neural_Network implements Serializable {
|
||||
// VARIABILE DI CLASSE:
|
||||
|
||||
// La variabile griglia è un array bidimensionale:ogni riga è un layer di
|
||||
// neuroni
|
||||
// ad ognuno dei quali è associato un vettore di pesi di lughezza uguale al
|
||||
// numero
|
||||
// di neuroni nella riga successiva.
|
||||
|
||||
// N.B.:nel layer di input e in quelli intermedi viene creato un neurone "in
|
||||
// più"
|
||||
// che funge da "bias" per quelli successivi (ha uscita sempre uguale ad uno
|
||||
// ma
|
||||
// pesi diversi per ogni neurone a cui si collega).
|
||||
|
||||
Neuron[][] griglia;
|
||||
static final long serialVersionUID = 1;
|
||||
// originale = 1.2
|
||||
double soglia = 0.001;
|
||||
double maxcycle = 1000;
|
||||
double acceptanceThr = 0.5;
|
||||
|
||||
public void setThreshold(double soglia) {
|
||||
this.soglia = soglia;
|
||||
}
|
||||
|
||||
public void setAcceptanceThreshold(double treshold) {
|
||||
this.acceptanceThr = treshold;
|
||||
}
|
||||
|
||||
public void setCycles(double cycs) {
|
||||
this.maxcycle = cycs;
|
||||
}
|
||||
|
||||
public static enum ACTIVATIONFUNCTION {
|
||||
HEAVYSIDE, SIGMOID, IDENTITY
|
||||
}
|
||||
|
||||
public Neural_Network(int N, int M, ACTIVATIONFUNCTION attifun) {
|
||||
this(N, M, attifun.ordinal() + 1);
|
||||
}
|
||||
|
||||
public Neural_Network(int N, int M, ACTIVATIONFUNCTION attifun, float[] V) {
|
||||
this(N, M, attifun.ordinal() + 1, V);
|
||||
}
|
||||
|
||||
public static double[] preprocessObjects(Object[] vector) {
|
||||
|
||||
double[] out = new double[vector.length];
|
||||
|
||||
for (int i = 0; i < vector.length; i++) {
|
||||
double element = 0;
|
||||
if (vector[i] != null)
|
||||
element = Double.parseDouble("" + vector[i]);
|
||||
|
||||
if (element == 0)
|
||||
element = 0.1;
|
||||
|
||||
out[i] = element;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
public static double[] preprocessObjects(double[] vector) {
|
||||
|
||||
double[] out = new double[vector.length];
|
||||
|
||||
for (int i = 0; i < vector.length; i++) {
|
||||
double element = vector[i];
|
||||
if (element == 0)
|
||||
element = 0.1;
|
||||
|
||||
out[i] = element;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
public double[] getPositiveCase() {
|
||||
double[] out = new double[0];
|
||||
|
||||
if (griglia.length > 0) {
|
||||
out = new double[griglia[griglia.length - 1].length];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = 1f;
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
public double[] getNegativeCase() {
|
||||
double[] out = new double[0];
|
||||
|
||||
if (griglia.length > 0) {
|
||||
out = new double[griglia[griglia.length - 1].length];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
public static int[] setupInnerLayers(int... numberOfNeurons) {
|
||||
int[] layers = null;
|
||||
|
||||
if (numberOfNeurons.length > 0) {
|
||||
layers = new int[numberOfNeurons.length];
|
||||
|
||||
for (int i = 0; i < numberOfNeurons.length; i++) {
|
||||
layers[i] = numberOfNeurons[i];
|
||||
}
|
||||
}
|
||||
return layers;
|
||||
}
|
||||
|
||||
public Neural_Network(int N, int M, int[] t, ACTIVATIONFUNCTION attifun) {
|
||||
this(N, M, t, attifun.ordinal() + 1);
|
||||
}
|
||||
|
||||
// COSTRUTTORI:
|
||||
|
||||
// Costruttore della rete nel caso di assenza di nodi neuroni nascosti
|
||||
// :costruisce
|
||||
// una rete neurale feed forward ad uno stato.in ingresso bisogna
|
||||
// specificare la
|
||||
// funzione di attivazione dei neuroni di output
|
||||
|
||||
public Neural_Network(int N, int M, int attifun) {
|
||||
// N è il numero di neuroni di input
|
||||
// M è il numero di neuroni di output
|
||||
// attifun è il codice della funzione di attivazione di un neurone
|
||||
this.griglia = new Neuron[2][];
|
||||
|
||||
// creazione dei neuroni di input:sono neuroni aventi l'identità come
|
||||
// funzione di attivazione
|
||||
Neuron[] input = new Neuron[N + 1];
|
||||
input[0] = new Neuron(M, 4);
|
||||
for (int i = 1; i < N + 1; i++) {
|
||||
input[i] = new Neuron(M, 3);
|
||||
}
|
||||
// La prima riga di "griglia" viene uguagliata al vettore di neuroni
|
||||
// appena creato
|
||||
this.griglia[0] = input;
|
||||
|
||||
// creazione dei neuroni di output:sono neuroni aventi la corrispondente
|
||||
// ad "attifun" come funzione di attivazione
|
||||
Neuron[] output = new Neuron[M];
|
||||
for (int i = 0; i < M; i++) {
|
||||
output[i] = new Neuron(0, attifun);
|
||||
}
|
||||
this.griglia[1] = output;
|
||||
}// fine primo costruttore
|
||||
|
||||
// Secondo costruttore:E' uguale a quello i prima ma permette all'utente di
|
||||
// inizializzare a piacimento il vettore dei pesi dell'input
|
||||
|
||||
public Neural_Network(int N, int M, int attifun, float[] V) {
|
||||
griglia = new Neuron[2][];
|
||||
// creazione dei neuroni di input:
|
||||
Neuron[] input = new Neuron[N + 1];
|
||||
input[0] = new Neuron(M, 4);
|
||||
for (int i = 1; i < N + 1; i++) {
|
||||
input[i] = new Neuron(M, 3, V);
|
||||
}
|
||||
this.griglia[0] = input;
|
||||
// creazione dei neuroni di output:
|
||||
Neuron[] output = new Neuron[M];
|
||||
for (int i = 0; i < M; i++) {
|
||||
output[i] = new Neuron(0, attifun);
|
||||
}
|
||||
this.griglia[1] = output;
|
||||
}// fine secondo costruttore
|
||||
|
||||
// Terzo costruttore,caso di più strati:viene generata una rete con un
|
||||
// numero di
|
||||
// strati nascosti e un numero di neuroni per ogni strato,stabiliti
|
||||
// dall'utente
|
||||
|
||||
// Il vettore "t", in ingresso,ha come lunghezza il numero di strati interni
|
||||
// e il suo
|
||||
// i-esimo elemento è un numero intero che indica quanti neuroni ci sono
|
||||
// nell'i-esimo
|
||||
// layer nascosto della rete.
|
||||
public Neural_Network(int N, int M, int[] t, int attifun) {
|
||||
|
||||
griglia = new Neuron[t.length + 2][];
|
||||
// t.length è il numero di strati nascosti a cui vanno aggiunti quello
|
||||
// di input e
|
||||
// di output
|
||||
|
||||
// creazione dei neuroni di input:come nel primo costruttore
|
||||
Neuron[] input = new Neuron[N + 1];
|
||||
input[0] = new Neuron(t[0], 4);
|
||||
for (int i = 1; i < N + 1; i++) {
|
||||
input[i] = new Neuron(t[0], 3);
|
||||
}
|
||||
this.griglia[0] = input;
|
||||
|
||||
// creazione dei neuroni hidden:hanno una funzione di attivazione uguale
|
||||
// a quella degli output
|
||||
// N.B.:per ogni strato nascosto viene generato anche un neurone "bias"
|
||||
Neuron[] aux;
|
||||
for (int i = 0; i < t.length; i++) {
|
||||
aux = new Neuron[t[i] + 1];
|
||||
if (i != t.length - 1) {
|
||||
// creazione di un neurone bias al posto 0
|
||||
aux[0] = new Neuron(t[i + 1], 4);
|
||||
// creazione del resto dei neuroni
|
||||
for (int g = 1; g < (t[i] + 1); g++) {
|
||||
aux[g] = new Neuron(t[i + 1], attifun);
|
||||
}
|
||||
} else {
|
||||
// neurone bias
|
||||
aux[0] = new Neuron(M, 4);
|
||||
for (int j = 1; j < t[i] + 1; j++) {
|
||||
// creazione del resto dei neuroni nel layer precedente a
|
||||
// quello di output
|
||||
aux[j] = new Neuron(M, attifun);
|
||||
}
|
||||
}
|
||||
|
||||
this.griglia[i + 1] = aux;
|
||||
}
|
||||
|
||||
// creazione dei neuroni di output:
|
||||
Neuron[] output = new Neuron[M];
|
||||
for (int i = 0; i < M; i++) {
|
||||
output[i] = new Neuron(0, attifun);
|
||||
}
|
||||
this.griglia[t.length + 1] = output;
|
||||
|
||||
}
|
||||
|
||||
// METODI DI ISTANZA
|
||||
|
||||
/*
|
||||
* Il metodo "propagate" propaga l'input della rete neurale fino ai neuroni di output.Fa uso del metodo prop
|
||||
*/
|
||||
public double[] propagate(double[] input) {
|
||||
if (input.length == griglia[0].length - 1)
|
||||
return prop(input, 0);
|
||||
else
|
||||
|
||||
System.out.println("Errore:numero di input non valido! ");
|
||||
return null;
|
||||
}
|
||||
|
||||
/*
|
||||
* Espleta le funzioni di propagate,ma ha in ingresso anche un intero,che rappresenta lo strato in analisi ed utile per la ricorsione
|
||||
*/
|
||||
private double[] prop(double[] input, int i) {
|
||||
double multip;
|
||||
// la variabile multip è il risultato della somma pesata degli output
|
||||
// dei neuroni "superiori"
|
||||
double[] arrayaux;
|
||||
// arrayaux è un vettore contenente gli input a tutti i neuroni del
|
||||
// layer i+1-esimo,
|
||||
// calcolati come somme pesate degli output dei neuroni dello strato
|
||||
// i-esimo.
|
||||
|
||||
if (griglia[i][0].W.length != 0) {
|
||||
arrayaux = new double[griglia[i][0].W.length];
|
||||
for (int j = 0; j < griglia[i][0].W.length; j++)
|
||||
|
||||
{
|
||||
multip = griglia[i][0].W[j];
|
||||
for (int g = 1; g < griglia[i].length; g++) {
|
||||
multip += griglia[i][g].W[j] * griglia[i][g].generaOutput(input[g - 1]);
|
||||
}
|
||||
arrayaux[j] = multip;
|
||||
}
|
||||
// chiamata ricorsiva:il vettore arrayaux è l'input per lo strato
|
||||
// i+1-esimo
|
||||
return prop(arrayaux, i + 1);
|
||||
} else
|
||||
// questo è il caso in cui siamo giunti all'ultimo strato della rete e
|
||||
// dunque il metodo
|
||||
// deve restituire le uscite dei neuroni di output
|
||||
{
|
||||
arrayaux = new double[griglia[i].length];
|
||||
for (int j = 0; j < griglia[i].length; j++)
|
||||
arrayaux[j] = griglia[i][j].generaOutput(input[j]);
|
||||
|
||||
return arrayaux;
|
||||
}
|
||||
|
||||
}// fine metodo prop
|
||||
|
||||
/*
|
||||
* Il metodo trainPropagate è una variante di propagate sfruttata nell'addestramento della rete:propaga l'input di training fino ai neuroni di output generando, inoltre , per ogni strato "i", un vettore contenente le entrate dei neuroni dello strato i+1-esimo. I vari vettori prodotti vengono, alla fine, inseriti in un array bidimensionale
|
||||
*/
|
||||
// N.B.:anche qui si fa uso di un "sottometodo", trainprop,a fini
|
||||
// implementativi
|
||||
private double[][] trainpropagate(double[] input) {
|
||||
double[][] arrayout = new double[griglia.length + 1][];
|
||||
if (input.length == griglia[0].length - 1)
|
||||
// nella prima riga del vettore restituito da questo metodo c'è l'input
|
||||
// della rete
|
||||
{
|
||||
arrayout[0] = input;
|
||||
for (int i = 0; i < griglia.length; i++)
|
||||
// nelle altre righe ci sono i risultati di trainpropagate.
|
||||
arrayout[i + 1] = trainprop(arrayout[i], i);
|
||||
// nell'ultima riga ci sono i valori dei neuroni di output
|
||||
return arrayout;
|
||||
} else
|
||||
System.out.println("Errore:numero di input non valido! ");
|
||||
return null;
|
||||
}// fine metodo trainpropagate
|
||||
|
||||
/*
|
||||
* Genera un vettore contenente gli input ad ogni neurone dell'i+1-esimo layer
|
||||
*/
|
||||
private double[] trainprop(double[] input, int i) {
|
||||
// lo schema è simile a quello del metodo "prop"
|
||||
double multip;
|
||||
double[] arrayaux;
|
||||
|
||||
if (griglia[i][0].W.length != 0) {
|
||||
arrayaux = new double[griglia[i][0].W.length];
|
||||
for (int j = 0; j < griglia[i][0].W.length; j++)
|
||||
|
||||
{
|
||||
multip = griglia[i][0].W[j];
|
||||
for (int g = 1; g < griglia[i].length; g++) {
|
||||
multip += griglia[i][g].W[j] * griglia[i][g].generaOutput(input[g - 1]);
|
||||
}
|
||||
arrayaux[j] = multip;
|
||||
}
|
||||
|
||||
return arrayaux;
|
||||
} else {
|
||||
arrayaux = new double[griglia[i].length];
|
||||
for (int j = 0; j < griglia[i].length; j++)
|
||||
arrayaux[j] = griglia[i][j].generaOutput(input[j]);
|
||||
|
||||
return arrayaux;
|
||||
}
|
||||
}// fine trainprop
|
||||
|
||||
// BACK PROPAGATION:
|
||||
|
||||
/*
|
||||
* Il metodo BProp avvia l'algoritmo di Back Propagation per la correzione dei pesi della rete.Il metodo è dichiarato private perchè all'interno del programma funge da metodo ausiliario per "train"
|
||||
*/
|
||||
|
||||
private void BProp(double[] input, double[] realvalues) {
|
||||
|
||||
// Variabili del metodo:
|
||||
double[][] Ai = trainpropagate(input);
|
||||
// Ai contenente gli input di tutti i neuroni, strato per strato, ed
|
||||
// anche gli output
|
||||
// della rete
|
||||
int lungtrain = Ai.length;
|
||||
// lungtrain registra la lunghezza di Ai
|
||||
double[][] arraydelta = new double[griglia.length - 1][];
|
||||
// arraydelta contiene le derivate (i "delta") dell'errore di
|
||||
// approssimazione
|
||||
// rispetto agli ingressi dei neuroni dal primo strato nascosto fino
|
||||
// agli output
|
||||
// (i "delta" degli input non servono ai fini della Backpropagation)
|
||||
int lungdelta = this.griglia.length - 1;
|
||||
// il numero di righe di arraydelta è di un'unità inferiore a quello
|
||||
// della griglia
|
||||
|
||||
// Corpo del metodo:
|
||||
// l'ultima riga di arraydelta viene definita come un vettore di
|
||||
// lunghezza pari al
|
||||
// numero di output,infatti c'è un "delta" per ogni uscita
|
||||
arraydelta[lungdelta - 1] = new double[Ai[lungtrain - 1].length];
|
||||
// si procede, ora, al calcolo effettivo dei "delta" di output:
|
||||
|
||||
for (int i = 0; i < Ai[lungtrain - 1].length; i++) {
|
||||
double Yk = Ai[lungtrain - 1][i];// uscita dell'i-esimo neurone
|
||||
// di output
|
||||
double Ak = Ai[lungtrain - 2][i];// ingresso dell'i-esimo neurone
|
||||
// di output
|
||||
double Tk = realvalues[i];// valore atteso dell'i-esimo neurone di
|
||||
// output
|
||||
// forma analitica del "delta" dell'i-esimo neurone di output:
|
||||
double Dk = (Yk - Tk) * (1 / (1 + Math.exp(-1 * Ak))) * (1 - ((1 / (1 + Math.exp(-1 * Ak)))));
|
||||
arraydelta[lungdelta - 1][i] = Dk;// il "delta" viene posto al
|
||||
// posto i-esimo dell'ultima
|
||||
// riga di arraydelta
|
||||
}
|
||||
|
||||
// Gli altri "delta" sono calcolati a partire da quelli di output in una
|
||||
// "propagazione all'indietro":
|
||||
// "g" è l'indice dello strato della rete in analisi
|
||||
// "j" è l'indice del neurone in analisi nello strato g
|
||||
// "k" scorre sugli indici dei neuroni nello strato successivo
|
||||
for (int g = lungdelta - 2; g >= 0; g--) {
|
||||
arraydelta[g] = new double[Ai[g + 1].length];
|
||||
|
||||
double[] DKnext = arraydelta[g + 1];
|
||||
for (int j = 0; j < Ai[g + 1].length; j++) {
|
||||
double Ak = Ai[g + 1][j];
|
||||
double somma = 0;
|
||||
for (int k = 0; k < arraydelta[g + 1].length; k++) {
|
||||
// calcolo di una somma pesata dei "delta" dello stato
|
||||
// successivo,utile per la valutazione del
|
||||
// delta del neurone j-esimo
|
||||
somma += griglia[g + 1][j + 1].W[k] * DKnext[k];
|
||||
}
|
||||
// valutazione del delta del neurone j-esimo:
|
||||
double Dk = somma * (1 / (1 + Math.exp(-1 * Ak))) * (1 - ((1 / (1 + Math.exp(-1 * Ak)))));
|
||||
// il "delta" viene posto al posto j-esimo della riga g-esima di
|
||||
// arraydelta
|
||||
arraydelta[g][j] = Dk;
|
||||
}
|
||||
}
|
||||
|
||||
// Correzioni per i pesi tramite i delta calcolati:
|
||||
for (int g = 0; g < griglia.length - 1; g++) {
|
||||
float[] D = new float[Ai[g + 1].length];
|
||||
// si mettono in D i delta dello strato g+1
|
||||
for (int k = 0; k < Ai[g + 1].length; k++) {
|
||||
D[k] = (float) (0.5f * arraydelta[g][k]);
|
||||
}
|
||||
// si aggiornano i pesi dei neuroni di input tramite D
|
||||
griglia[g][0].aggiornaPesi(D);
|
||||
// si aggiornano i pesi dei neuroni strato per strato
|
||||
for (int i = 1; i < griglia[g].length; i++) {
|
||||
float[] V = new float[Ai[g + 1].length];
|
||||
|
||||
for (int k = 0; k < Ai[g + 1].length; k++) {
|
||||
V[k] = (float) (0.5f * (griglia[g][i].generaOutput(Ai[g][i - 1]) * arraydelta[g][k]));
|
||||
}
|
||||
|
||||
griglia[g][i].aggiornaPesi(V);
|
||||
}
|
||||
}
|
||||
}// Fine Back Propagation
|
||||
|
||||
// Il metodo "train" avvia effettivamente l'addestramento sfruttando la Back
|
||||
// Propagation
|
||||
public void train(double[][] inputvet, double[][] correctoutputvet) {
|
||||
/*
|
||||
* il metodo ha bisogno di due ingressi: inputvet:un vettore le cui righe sono gli elementi del training set (ad esempio un vettore di features per ogni elemento del Training Set) correctoutputvet:un vettore le cui righe sono i valori attesi di uscita della rete per ogni elemento del Training Set
|
||||
*/
|
||||
|
||||
// c'è innanzitutto un controllo che il numero degli output attesi
|
||||
// corrisponda al
|
||||
// numero di neuroni di uscita della rete
|
||||
if (griglia[griglia.length - 1].length != correctoutputvet[0].length)
|
||||
System.out.println("Errore: il vettore degli output " + "NON ha una lunghezza pari " + "a quella dell'output " + "della rete");
|
||||
else {
|
||||
// Si avvia la backpropagation 1000 volte aggiornando i pesi ogni
|
||||
// volta in base
|
||||
// agli elementi del training set
|
||||
// La tecnica di aggiornamento dei pesi è di tipo "On-Line":questi
|
||||
// vengono modificati
|
||||
// secondo le derivate dell'errore ad ogni input di addestramento
|
||||
// introdotto.
|
||||
/* for (int j = 0;j < 1000;j++) */
|
||||
double en = 2;
|
||||
int counter = 0;
|
||||
while ((en > soglia) && (counter <= maxcycle)) {
|
||||
en = 0;
|
||||
for (int i = 0; i < inputvet.length; i++) {
|
||||
/*
|
||||
* double[] arraux = new double[22]; double[] arraux2 = new double[7]; for (int g = 0; g < 22; g++) { arraux[g] = inputvet[i][g];} for (int g = 0; g < 7; g++) { arraux2[g] = correctoutputvet[i][g]; } this.BProp(arraux, arraux2); en += energy(this.propagate(arraux),arraux2); }
|
||||
*/
|
||||
this.BProp(inputvet[i], correctoutputvet[i]);
|
||||
en += energy(this.propagate(inputvet[i]), correctoutputvet[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* System.out.println("inputvet... "); for (int j = 0;j<22;j++){ System.out.print(inputvet[1][j] + " ");} System.out.println();
|
||||
*
|
||||
* System.out.println("inputvet prop... "); for (int j = 0;j<7;j++){ System.out.print(this.propagate(inputvet[1])[j] + " ");} System.out.println();
|
||||
*
|
||||
* System.out.println("correct output... "); for (int j = 0;j<7;j++){ System.out.print(correctoutputvet[1][j] + " ");} System.out.println();
|
||||
*/
|
||||
|
||||
System.out.println("errore: " + en);
|
||||
counter++;
|
||||
}
|
||||
System.out.println("Scarto Finale: " + en);
|
||||
if (counter >= maxcycle)
|
||||
System.out.println("training incompleto: non sono riuscito ridurre l'errore sotto la soglia!");
|
||||
else
|
||||
System.out.println("training completo!");
|
||||
}
|
||||
}// fine metodo train
|
||||
|
||||
/*
|
||||
* Il metodo "energy" valuta l'errore quadratico tra due vettori (nel nostro caso l'output della rete e quello atteso)
|
||||
*/
|
||||
private double energy(double[] vettore1, double[] vettore2) {
|
||||
double nrg = (float) Math.pow((vettore1[0] - vettore2[0]), 2);
|
||||
for (int i = 1; i < vettore2.length; i++) {
|
||||
nrg = nrg + Math.pow((vettore1[i] - vettore2[i]), 2);
|
||||
}
|
||||
return (float) (0.5 * nrg);
|
||||
}// fine metodo energy
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Il metodo "writeout" stampa a video il numero "1" se l'uscita di un neurone supera il valore di "soglia" e stampa "0" se questo non succede
|
||||
*/
|
||||
public void writeout(double numero, double soglia) {
|
||||
if (numero < soglia)
|
||||
System.out.println("Uscita : " + 0);
|
||||
else
|
||||
System.out.println("Uscita : " + 1);
|
||||
}
|
||||
|
||||
//classify
|
||||
public double[] getClassification(double[] out){
|
||||
double[] o = new double[out.length];
|
||||
for (int i=0;i<out.length;i++){
|
||||
if (out[i]<acceptanceThr)
|
||||
o[i] = 0;
|
||||
else
|
||||
o[i] = 1;
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
// /Il main serve solo a testare la classe.
|
||||
public static void main(String[] args) {
|
||||
int[] t = { 2 };
|
||||
// float[] V = {0.6f};
|
||||
// NN(numero input,numero output,array layers,numero funzatt)
|
||||
// Neural_Network nn = new Neural_Network(2,1,t,2);
|
||||
Neural_Network nn = new Neural_Network(2, 1, t, 2);
|
||||
double[] input = { 1, 1 };
|
||||
double[] output = { 0 };
|
||||
double[] input1 = { 0.1, 0.1 };
|
||||
double[] output1 = { 0 };
|
||||
double[] input2 = { 1, 0.1 };
|
||||
double[] output2 = { 1 };
|
||||
double[] input3 = { 0.1, 1 };
|
||||
double[] output3 = { 1 };
|
||||
double[][] in = new double[4][];
|
||||
double[][] out = new double[4][];
|
||||
in[0] = input;
|
||||
in[1] = input1;
|
||||
out[0] = output;
|
||||
out[1] = output1;
|
||||
in[2] = input2;
|
||||
out[2] = output2;
|
||||
in[3] = input3;
|
||||
out[3] = output3;
|
||||
|
||||
nn.train(in, out);
|
||||
double[] dummy = { 0, 0 };
|
||||
System.out.println("responso sul dummy: " + nn.propagate(dummy)[0]);
|
||||
|
||||
nn.writeout(nn.propagate(dummy)[0], 0.5);
|
||||
/*
|
||||
* nn.writeout(nn.propagate(input1)[0],0.5); nn.writeout(nn.propagate(input2)[0],0.5); nn.writeout(nn.propagate(input3)[0],0.5); System.out.println("Valore vero: " + nn.propagate(input)[0]); System.out.println("Valore vero: " + nn.propagate(input1)[0]); System.out.println("Valore vero: " + nn.propagate(input2)[0]); System.out.println("Valore vero: " + nn.propagate(input3)[0]);
|
||||
*/
|
||||
}// Fine Main
|
||||
|
||||
}// Fine Classe Neural_Network
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
//Classe Neurone:le sue istanze sono neuroni "artificiali" con funzione di uscita
|
||||
//a scelta tra gradino,sigmoide o identità.
|
||||
|
||||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks;
|
||||
import java.io.Serializable;
|
||||
import java.util.Random;
|
||||
|
||||
public class Neuron implements Serializable{
|
||||
|
||||
public static final long serialVersionUID=1;
|
||||
//VARIABILI D'ISTANZA:
|
||||
|
||||
//attivfun è un numero intero corrispondente al tipo di funzione di attivazione
|
||||
//dei neuroni interni e di output
|
||||
public int attivfun;
|
||||
//W è un vettore contenente i pesi delle connessioni con i neuroni successivi
|
||||
public float[] W;
|
||||
|
||||
//COSTRUTTORI:
|
||||
|
||||
/*Primo costruttore della classe:viene richiesto in ingresso il numero N i neuroni
|
||||
ai quali si lega il neurone da costruire e il numero corrispondente alla funzione
|
||||
di attivazione (per la corrispondenza numero-funzione vedere dopo) */
|
||||
|
||||
public Neuron (int N,int numero){
|
||||
attivfun = numero;
|
||||
W = new float[N];
|
||||
|
||||
//I pesi sono inizializzati a valori random tra -1 e 1
|
||||
|
||||
Random rand = new Random();
|
||||
for (int i = 0; i < N ;i++) {
|
||||
W[i] = 2*rand.nextFloat()-1;} //0.5f;
|
||||
}
|
||||
|
||||
/*Altro costruttore che permette all'utente di scegliere
|
||||
il vettore dei pesi inziale */
|
||||
public Neuron (int N,int numero,float[] V){
|
||||
attivfun = numero;
|
||||
W = new float[N];
|
||||
|
||||
if (V.length == N){
|
||||
for (int i = 0; i < N ;i++) {
|
||||
W[i] = V[i];}
|
||||
}
|
||||
else
|
||||
System.out.println("Errore:La lunghezza del vettore dei pesi introdotta non è corretta! ");
|
||||
}
|
||||
|
||||
|
||||
//METODI DI CLASSE
|
||||
|
||||
/*Il metodo generaOutput ha in ingresso l'input giunto al neurone
|
||||
sommando i contributi dei neuroni precedenti.In uscita fornisce l'output del
|
||||
neurone calcolato in base alla funzione di attivazione scelta dall'utente */
|
||||
|
||||
public double generaOutput(double input) {
|
||||
/*Il numero 1 corrisponde ad una funzione di attivazione di tipo
|
||||
"heavyside" */
|
||||
if (attivfun == 1)
|
||||
{if (input > 0) return 1;
|
||||
else return 0;}
|
||||
//Il numero 2 corrisponde alla sigmoide
|
||||
else
|
||||
if (attivfun == 2)
|
||||
{return (1/(1 + Math.exp(-1 * input)));}
|
||||
/*Il numero 3 corrisponde ad un output uguale all'input (identità).
|
||||
E' il caso dei neuroni di input */
|
||||
if (attivfun == 3)
|
||||
{return input;}
|
||||
else
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*Il metodo aggiornaPesi aggiorna il vecchio vettore dei pesi del neurone in questione
|
||||
decrementandone i valori con quelli del vettore in ingresso.E' assunto che il
|
||||
vettore di ingresso abbia lunghezza uguale a quello dei pesi */
|
||||
|
||||
public void aggiornaPesi(float[] V) {
|
||||
for(int i = 0;i < V.length;i++)
|
||||
W[i] -= V[i];
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
///Il main serve solo a testare la classe.
|
||||
|
||||
public static void main(String[] args) {
|
||||
Neuron neuron1 = new Neuron(4,1);
|
||||
System.out.println("con la sigmoide: "+ neuron1.generaOutput(1));
|
||||
System.out.println("con heaviside: " + neuron1.generaOutput(1));
|
||||
System.out.println("heaviside e input negativo: " + neuron1.generaOutput(-12));
|
||||
System.out.println("unitaria e input positivo: " + neuron1.generaOutput(12));
|
||||
float[] G = new float[4];
|
||||
G[0] = 0.1F;
|
||||
G[1] = 0.2F;
|
||||
G[2] = 0.1F;
|
||||
G[3] = 2F;
|
||||
neuron1.aggiornaPesi(G);
|
||||
for (int i = 0; i < G.length;i++)
|
||||
System.out.println(neuron1.W[i]);
|
||||
}
|
||||
|
||||
}//fine classe
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
// this class reads one line at a time and returns it as a string
|
||||
public class LineReader
|
||||
{
|
||||
String[] column;
|
||||
FileInputStream fis;
|
||||
BufferedInputStream bis;
|
||||
|
||||
// constructor
|
||||
public LineReader (String path) {
|
||||
try {
|
||||
fis = new FileInputStream(path);
|
||||
bis = new BufferedInputStream(fis);
|
||||
}
|
||||
catch (IOException e) {}
|
||||
/*
|
||||
this.inputFile = new File(path);
|
||||
try {in = new FileReader(inputFile);} catch (IOException e) {}
|
||||
*/
|
||||
}
|
||||
|
||||
// read the next line, split it and return the parts as a string array
|
||||
public boolean NextLineSplitted () {
|
||||
column = null;
|
||||
column = NextLine().split(";");
|
||||
if (column[0] != "#EOF#") {
|
||||
for (int i = 0; i < column.length; i++) {
|
||||
column[i] = column[i].trim();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
else{return false;}
|
||||
}
|
||||
|
||||
// read the next line, return the line as string
|
||||
public String NextLine() {
|
||||
int i;
|
||||
char[] temp_array = new char[50000];
|
||||
char[] temp_array2;
|
||||
boolean last_line;
|
||||
int counter;
|
||||
String temp_line = "";
|
||||
|
||||
do {
|
||||
temp_array2 = null;
|
||||
counter = 0;
|
||||
last_line = true;
|
||||
// read a line
|
||||
try {
|
||||
while ( (i = bis.read()) != -1 ) {
|
||||
last_line = false;
|
||||
if (i == 13 || i == 10) {
|
||||
break;
|
||||
}
|
||||
else if( i != 10 && i != 13) {
|
||||
temp_array[counter++] = (char)i;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (IOException e) {}
|
||||
// put the array into a string
|
||||
if (last_line) {
|
||||
temp_line = "#EOF#";
|
||||
}
|
||||
else if (counter != 0) {
|
||||
temp_array2 = new char[counter];
|
||||
boolean all_spaces = true;
|
||||
for (int j = 0; j < counter; j++) {
|
||||
if (temp_array[j] != ' ') {all_spaces = false;}
|
||||
temp_array2[j] = temp_array[j];
|
||||
}
|
||||
if (all_spaces) {temp_line = "";}
|
||||
else {temp_line = new String(temp_array2);}
|
||||
if (temp_line.length() >= 2 && temp_line.charAt(0) == '/' && temp_line.charAt(1) == '/') {
|
||||
temp_line = "";
|
||||
}
|
||||
}
|
||||
else {
|
||||
temp_line = "";
|
||||
}
|
||||
|
||||
} while (temp_line == "");
|
||||
return temp_line.trim();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,534 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
|
||||
public class NeuralNet implements Serializable {
|
||||
|
||||
Neuron[] neurons;
|
||||
Synapse[] synapses;
|
||||
int nolayers; // no of layers, inc. input and output layers
|
||||
Layer[] layers;
|
||||
private Randomizer randomizer;
|
||||
|
||||
// constructor
|
||||
// opens the configuration file and creates a net according to it.
|
||||
public NeuralNet (String path, Randomizer randomizer) {
|
||||
this.randomizer = randomizer;
|
||||
LineReader linereader = new LineReader(path);
|
||||
while (linereader.NextLineSplitted()){
|
||||
// if it declares # of objects, dimension the appropriate array
|
||||
if (linereader.column[0].compareTo("#neurons") == 0) { neurons = new Neuron[Integer.parseInt(linereader.column[1])]; }
|
||||
if (linereader.column[0].compareTo("#synapses") == 0) { synapses = new Synapse[Integer.parseInt(linereader.column[1])]; }
|
||||
// if it represents an input neuron, create a neuron object
|
||||
if (linereader.column[0].compareTo("i") == 0) { neurons[Integer.parseInt(linereader.column[1])] = new Neuron(Integer.parseInt(linereader.column[1])); }
|
||||
// if it represents a neuron, create a neuron object
|
||||
if (linereader.column[0].compareTo("n") == 0) { neurons[Integer.parseInt(linereader.column[1])] = new Neuron( Integer.parseInt(linereader.column[1]), Integer.parseInt(linereader.column[2]), Double.parseDouble(linereader.column[3]), linereader.column[4].charAt(0), Double.parseDouble(linereader.column[5]), Double.parseDouble(linereader.column[6]), randomizer ); }
|
||||
// if it represents a synapse, create a synapse object
|
||||
if (linereader.column[0].compareTo("s") == 0) { synapses[Integer.parseInt(linereader.column[1])] =
|
||||
new Synapse(
|
||||
neurons[Integer.parseInt(linereader.column[2])],
|
||||
neurons[Integer.parseInt(linereader.column[3])],
|
||||
randomizer
|
||||
); }
|
||||
}
|
||||
linereader = null;
|
||||
// first find out how many layers there are
|
||||
int temp_maxlayer = 0;
|
||||
for (int i = 0; i < neurons.length; i++) {
|
||||
if (neurons[i].layer > temp_maxlayer) {temp_maxlayer = neurons[i].layer;}
|
||||
}
|
||||
nolayers = temp_maxlayer+1;
|
||||
// then create layer objects
|
||||
layers = new Layer[nolayers];
|
||||
for (int i = 0; i < nolayers; i++) {layers[i] = new Layer(i);}
|
||||
NeuronsInOut();
|
||||
}
|
||||
|
||||
public static double[] preprocessObjects(Object[] vector) {
|
||||
|
||||
double[] out = new double[vector.length];
|
||||
|
||||
for (int i = 0; i < vector.length; i++) {
|
||||
double element = 0;
|
||||
if (vector[i] != null)
|
||||
element = Double.parseDouble("" + vector[i]);
|
||||
|
||||
out[i] = element;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
public double[] getNegativeCase() {
|
||||
double[] out = new double[0];
|
||||
|
||||
if (topology.length > 0) {
|
||||
out = new double[topology[topology.length - 1]];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = -1f;
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
public double[] getPositiveCase() {
|
||||
double[] out = new double[0];
|
||||
|
||||
if (topology.length > 0) {
|
||||
out = new double[topology[topology.length - 1]];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = 1f;
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
public static int[] setupInnerLayers(int... numberOfNeurons) {
|
||||
int[] layers = null;
|
||||
|
||||
if (numberOfNeurons.length > 0) {
|
||||
layers = new int[numberOfNeurons.length];
|
||||
|
||||
for (int i = 0; i < numberOfNeurons.length; i++) {
|
||||
layers[i] = numberOfNeurons[i];
|
||||
}
|
||||
}
|
||||
return layers;
|
||||
}
|
||||
|
||||
public NeuralNet(int N, int M, int[] t) {
|
||||
|
||||
Randomizer randomizer = new Randomizer();
|
||||
int[] noofneurons = null;
|
||||
double[] learnratecoeff = null;
|
||||
char[] axonfamily = null;
|
||||
double[] momentumrate = null;
|
||||
double[] flatness = null;
|
||||
|
||||
int len = 2;
|
||||
if (t!=null){
|
||||
len = len+t.length;
|
||||
}
|
||||
noofneurons = new int[len];
|
||||
learnratecoeff = new double[len];
|
||||
axonfamily = new char[len];
|
||||
momentumrate = new double[len];
|
||||
flatness = new double[len];
|
||||
|
||||
noofneurons[0] = N;
|
||||
learnratecoeff[0]=1;
|
||||
axonfamily[0] = 't';
|
||||
momentumrate[0] = 0;
|
||||
flatness[0] = 1;
|
||||
for (int i=1;i<len-1;i++){
|
||||
noofneurons[i]=t[i-1];
|
||||
learnratecoeff[i]=1;
|
||||
axonfamily[i] = 't';
|
||||
momentumrate[i] = 0.4;
|
||||
flatness[i] = 1;
|
||||
}
|
||||
|
||||
noofneurons[len-1] = M;
|
||||
learnratecoeff[len-1]=1;
|
||||
axonfamily[len-1] = 't';
|
||||
momentumrate[len-1] = 0.4;
|
||||
flatness[len-1] = 1;
|
||||
|
||||
init(noofneurons, learnratecoeff, axonfamily, momentumrate, flatness, randomizer);
|
||||
|
||||
}
|
||||
|
||||
int[] topology;
|
||||
|
||||
public void init (int[] noofneurons, double[] learningratecoefficient, char[] axonfamily, double[] momentumrate, double[] axonfuncflatness, Randomizer randomizer) {
|
||||
this.randomizer = randomizer;
|
||||
this.topology = noofneurons;
|
||||
int temp_nooflayers = noofneurons.length;
|
||||
nolayers = noofneurons.length;
|
||||
// determine the no of neurons and create the array
|
||||
int temp_noofneurons = 0;
|
||||
for ( int i = 0; i < temp_nooflayers; i++ ) {
|
||||
temp_noofneurons += noofneurons[i];
|
||||
}
|
||||
neurons = new Neuron[temp_noofneurons];
|
||||
// determine the no of synapses and create the array
|
||||
int temp_noofsynapses = 0;
|
||||
for ( int i = 0; i < temp_nooflayers-1; i++ ) {
|
||||
temp_noofsynapses += noofneurons[i] * noofneurons[i+1];
|
||||
}
|
||||
synapses = new Synapse[temp_noofsynapses];
|
||||
// instantiate neurons:
|
||||
int temp_neuronidcounter = 0;
|
||||
// first instantiate input neurons
|
||||
for ( int i = 0; i < noofneurons[0]; i++ ) {
|
||||
neurons[temp_neuronidcounter] = new Neuron(temp_neuronidcounter);
|
||||
temp_neuronidcounter++;
|
||||
}
|
||||
// then instantiate hidden and output neurons
|
||||
for ( int i = 1; i < temp_nooflayers; i++ ) {
|
||||
for ( int j = 0; j < noofneurons[i]; j++ ) {
|
||||
neurons[temp_neuronidcounter] = new Neuron(temp_neuronidcounter, i, axonfuncflatness[i], axonfamily[i], momentumrate[i], learningratecoefficient[i], randomizer);
|
||||
temp_neuronidcounter++;
|
||||
}
|
||||
}
|
||||
// then create layer objects
|
||||
layers = new Layer[temp_nooflayers];
|
||||
for (int i = 0; i < temp_nooflayers; i++) {layers[i] = new Layer(i);}
|
||||
// instantiate synapses
|
||||
int temp_synapseidcounter = 0;
|
||||
for ( int i = 0; i < temp_nooflayers-1; i++) {
|
||||
for ( int j = 0; j < layers[i].neurons.length; j++ ) {
|
||||
for ( int k = 0; k < layers[i+1].neurons.length; k++ ) {
|
||||
synapses[temp_synapseidcounter++] = new Synapse(layers[i].neurons[j], layers[i+1].neurons[k], randomizer);
|
||||
}
|
||||
}
|
||||
}
|
||||
NeuronsInOut();
|
||||
}
|
||||
|
||||
// another constructor. creates a MULTILAYER PERCEPTRON with
|
||||
// given no. of layers, no of neurons, learning rates, momentum parameters,
|
||||
// axonfamilies, flatness. except for noofneurons, all parameters are
|
||||
// ineffectual for the first layer.
|
||||
public NeuralNet (int[] noofneurons, double[] learningratecoefficient, char[] axonfamily, double[] momentumrate, double[] axonfuncflatness, Randomizer randomizer) {
|
||||
init( noofneurons, learningratecoefficient, axonfamily, momentumrate, axonfuncflatness, randomizer);
|
||||
}
|
||||
|
||||
// This method is used by constructors only.
|
||||
// It determines the incoming and outgoing neurons / synapses for each neuron
|
||||
// and set them in the neuron. This information is to be used later during feed forward and back propagation.
|
||||
private void NeuronsInOut () {
|
||||
// and then create neuronsin, neuronsout, synapsesin, synapsesout arrays in the neuron objects
|
||||
// in order to determine relationships between neurons
|
||||
Neuron[] temp_neuronsin;
|
||||
Neuron[] temp_neuronsout;
|
||||
Synapse[] temp_synapsesin;
|
||||
Synapse[] temp_synapsesout;
|
||||
|
||||
int incounter; int outcounter;
|
||||
for (int i = 0; i < neurons.length; i++) {
|
||||
// first determine the dimension of the arrays
|
||||
temp_neuronsin = null;
|
||||
temp_neuronsout = null;
|
||||
incounter = 0; outcounter = 0;
|
||||
for (int j = 0; j < synapses.length; j++) {
|
||||
if (synapses[j].sourceunit == neurons[i]) {outcounter++;}
|
||||
if (synapses[j].targetunit == neurons[i]) {incounter++;}
|
||||
}
|
||||
temp_neuronsin = new Neuron[incounter];
|
||||
temp_synapsesin = new Synapse[incounter];
|
||||
temp_neuronsout = new Neuron[outcounter];
|
||||
temp_synapsesout = new Synapse[outcounter];
|
||||
// then fill each array
|
||||
incounter = 0; outcounter = 0;
|
||||
for (int j = 0; j < synapses.length; j++) {
|
||||
if (synapses[j].sourceunit == neurons[i]) {
|
||||
temp_neuronsout[outcounter] = synapses[j].targetunit;
|
||||
temp_synapsesout[outcounter++] = synapses[j];
|
||||
}
|
||||
if (synapses[j].targetunit == neurons[i]) {
|
||||
temp_neuronsin[incounter] = synapses[j].sourceunit;
|
||||
temp_synapsesin[incounter++] = synapses[j];
|
||||
}
|
||||
}
|
||||
// set them in the neuron
|
||||
neurons[i].InsOuts(temp_neuronsin, temp_neuronsout, temp_synapsesin, temp_synapsesout);
|
||||
}
|
||||
}
|
||||
|
||||
// saves the configuration of the net to a file
|
||||
public void SaveConfig (String path) throws IOException {
|
||||
File outputFile = new File(path);
|
||||
FileWriter out = new FileWriter(outputFile);
|
||||
out.write("// Input units:\n");
|
||||
// no of neurons
|
||||
out.write("#neurons;"+neurons.length+"\n");
|
||||
out.write("// type;ID;layer;flatness;axonfamily;momentum;learningrate\n");
|
||||
// neurons
|
||||
for (int i = 0; i < neurons.length; i++) {
|
||||
if (neurons[i].layer == 0) {
|
||||
out.write("i;"+i+";0\n");
|
||||
}
|
||||
else {
|
||||
out.write("n;"+i+";"+neurons[i].layer+";"+neurons[i].axonfuncflatness+";"+neurons[i].axonfamily+";"+neurons[i].momentumrate+";"+neurons[i].learningratecoefficient+"\n");
|
||||
}
|
||||
}
|
||||
// synapses
|
||||
out.write("#synapses;"+synapses.length+"\n");
|
||||
out.write("// type; ID; sourceunit; targetunit\n");
|
||||
for (int i = 0; i < synapses.length; i++) {
|
||||
out.write("s;"+i+";"+synapses[i].sourceunit.id+";"+synapses[i].targetunit.id+"\n");
|
||||
}
|
||||
out.close();
|
||||
|
||||
}
|
||||
|
||||
// loads weights of the net from a file
|
||||
public void LoadWeights (String path) {
|
||||
LineReader linereader = new LineReader(path);
|
||||
while (linereader.NextLineSplitted()) {
|
||||
// if it's a synapse weight
|
||||
if (linereader.column[0].compareTo("w") == 0) { synapses[Integer.parseInt(linereader.column[1])].weight = Double.parseDouble(linereader.column[2]); }
|
||||
// if it's a neuron threshold
|
||||
if (linereader.column[0].compareTo("t") == 0) { neurons[Integer.parseInt(linereader.column[1])].threshold = Double.parseDouble(linereader.column[2]); }
|
||||
}
|
||||
linereader = null;
|
||||
}
|
||||
|
||||
// saves weights to a file
|
||||
public void SaveWeights (String path) throws IOException {
|
||||
File outputFile = new File(path);
|
||||
FileWriter out = new FileWriter(outputFile);
|
||||
// first write weight of each synapse
|
||||
for (int i = 0; i < synapses.length; i++) {
|
||||
out.write("w; "+i+"; "+synapses[i].weight+"\n");
|
||||
}
|
||||
out.write("\n");
|
||||
// then threshold of each neuron
|
||||
for (int i = 0; i < neurons.length; i++) {
|
||||
out.write("t; "+i+"; "+neurons[i].threshold+"\n");
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
// feeds the network forward and updates all the neurons.
|
||||
public void FeedForward (double[] inputs) {
|
||||
// feed input values
|
||||
for (int i = 0; i < layers[0].neurons.length; i++) {
|
||||
layers[0].neurons[i].output = inputs[i];
|
||||
}
|
||||
// begin from the first layer and propagate through layers.
|
||||
for (int i = 1; i < nolayers; i++) {
|
||||
// update the output of each neuron in this layer
|
||||
for (int j = 0; j < layers[i].neurons.length; j++) {
|
||||
layers[i].neurons[j].UpdateOutput();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// takes an array of input values, put them to the input neurons, feeds the net forward and returns the outputs of the output layer
|
||||
public double[] Output (double[] inputs) {
|
||||
FeedForward(inputs);
|
||||
double[] tempoutputs = new double[layers[nolayers-1].neurons.length];
|
||||
for(int i = 0; i < layers[nolayers-1].neurons.length; i++) {
|
||||
tempoutputs[i] = layers[nolayers-1].neurons[i].output;
|
||||
}
|
||||
return tempoutputs;
|
||||
}
|
||||
|
||||
// calculates a std error for this net using given cross validation patterns
|
||||
public double CrossValErrorRatio (PatternSet patternset) {
|
||||
int noofoutputunits = layers[nolayers-1].neurons.length;
|
||||
double[] abserrors = new double[noofoutputunits];
|
||||
for ( int i = 0; i < noofoutputunits; i++ ) { abserrors[i] = 0; }
|
||||
// calculate avg error for each neuron
|
||||
double errorratio = 0;
|
||||
double[] temp_output = new double[noofoutputunits];
|
||||
for (int j = 0; j < patternset.crossvalpatterns.length; j++) {
|
||||
temp_output = Output(patternset.crossvalpatterns[j].input);
|
||||
for (int i = 0; i < noofoutputunits; i++) {
|
||||
abserrors[i] += Math.abs( temp_output[i] - patternset.crossvalpatterns[j].target[i] );
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < noofoutputunits; i++) {
|
||||
abserrors[i] /= patternset.crossvalpatterns.length;
|
||||
errorratio += ( abserrors[i] / patternset.crossvaldeviations[i] );
|
||||
}
|
||||
errorratio /= noofoutputunits;
|
||||
return errorratio;
|
||||
}
|
||||
|
||||
// calculates a std error for this net using given test patterns
|
||||
public double TestErrorRatio (PatternSet patternset) {
|
||||
int noofoutputunits = layers[nolayers-1].neurons.length;
|
||||
double[] abserrors = new double[noofoutputunits];
|
||||
for ( int i = 0; i < noofoutputunits; i++ ) { abserrors[i] = 0; }
|
||||
// calculate avg error for each neuron
|
||||
double errorratio = 0;
|
||||
double[] temp_output = new double[noofoutputunits];
|
||||
for (int j = 0; j < patternset.testpatterns.length; j++) {
|
||||
temp_output = Output(patternset.testpatterns[j].input);
|
||||
for (int i = 0; i < noofoutputunits; i++) {
|
||||
abserrors[i] += Math.abs( temp_output[i] - patternset.testpatterns[j].target[i] );
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < noofoutputunits; i++) {
|
||||
abserrors[i] /= patternset.testpatterns.length;
|
||||
errorratio += ( abserrors[i] / patternset.testdeviations[i] );
|
||||
}
|
||||
errorratio /= noofoutputunits;
|
||||
return errorratio;
|
||||
}
|
||||
|
||||
// Training methods ¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/¯\_/
|
||||
|
||||
// takes all patterns one by one (with random order) and trains the net
|
||||
// using each one.
|
||||
public void IncrementalTrainPatterns(Pattern[] patterns, double rate) {
|
||||
int patternsnottrained = patterns.length; // no of patterns used
|
||||
int patterntotrain;
|
||||
int indexofpatterntotrain = -1;
|
||||
int counter;
|
||||
// turn all "selected" flags off
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
patterns[i].selected = false;
|
||||
}
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
patterntotrain = randomizer.random.nextInt(patternsnottrained);
|
||||
// find the index of the pattern to train
|
||||
counter = 0;
|
||||
for (int j = 0; j < patterns.length; j++) {
|
||||
if (!patterns[j].selected) {
|
||||
if (counter != patterntotrain) {
|
||||
counter++;
|
||||
}
|
||||
else if (counter == patterntotrain) {
|
||||
indexofpatterntotrain = j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// train the net using the selected pattern
|
||||
IncrementalTrain(rate, patterns[indexofpatterntotrain]);
|
||||
patterns[indexofpatterntotrain].selected = true;
|
||||
patternsnottrained--;
|
||||
}
|
||||
|
||||
// turn all "selected" flags off again
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
patterns[i].selected = false;
|
||||
}
|
||||
}
|
||||
|
||||
// trains the net incrementally.
|
||||
public void IncrementalTrain(double rate, Pattern pattern) {
|
||||
// feed fw
|
||||
FeedForward(pattern.input);
|
||||
// train the output layer first
|
||||
for (int j = 0; j < layers[nolayers-1].neurons.length; j++) {
|
||||
layers[nolayers-1].neurons[j].OutputIncrementalTrain(rate, pattern.target[j]);
|
||||
}
|
||||
// train hidden layers
|
||||
for (int i = nolayers-2; i > 0; i--) {
|
||||
for (int j = 0; j < layers[i].neurons.length; j++) {
|
||||
layers[i].neurons[j].HiddenIncrementalTrain(rate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// selects patterns (quantity: nopatterns) randomly and trains the net using those patterns.
|
||||
// repeats this until all patterns in the pattern array have been used for training
|
||||
public void MinibatchTrainPatterns(Pattern[] patterns, double rate, int nopatterns) {
|
||||
int patternsnottrained = patterns.length; // no of patterns used
|
||||
if (nopatterns > patterns.length) {nopatterns = patterns.length;}
|
||||
if (nopatterns < 1) {nopatterns = 1;}
|
||||
int patterntotrain;
|
||||
int noofpatternsselected;
|
||||
Pattern[] patternsselected;
|
||||
int indexofpatterntotrain = -1;
|
||||
int[] indexesofpatternstotrain = new int[nopatterns];
|
||||
int counter;
|
||||
|
||||
// turn all "selected" flags off
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
patterns[i].selected = false;
|
||||
}
|
||||
|
||||
while ( patternsnottrained > 0 ) {
|
||||
// choose patterns to be used for training and put them in the temp. pattern array
|
||||
noofpatternsselected = 0;
|
||||
while ( noofpatternsselected < nopatterns && patternsnottrained > 0 ) {
|
||||
patterntotrain = randomizer.random.nextInt(patternsnottrained);
|
||||
patternsnottrained--;
|
||||
// find the index of the pattern to be used
|
||||
counter = 0;
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
if (!patterns[i].selected) {
|
||||
if (counter != patterntotrain) {
|
||||
counter++;
|
||||
}
|
||||
else if (counter == patterntotrain) {
|
||||
indexofpatterntotrain = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
noofpatternsselected++;
|
||||
indexesofpatternstotrain[noofpatternsselected-1] = indexofpatterntotrain;
|
||||
patterns[indexofpatterntotrain].selected = true;
|
||||
}
|
||||
// train the net using the temp. pattern array
|
||||
patternsselected = null;
|
||||
patternsselected = new Pattern[noofpatternsselected];
|
||||
for (int i = 0; i < noofpatternsselected; i++) {
|
||||
patternsselected[i] = patterns[indexesofpatternstotrain[i]];
|
||||
}
|
||||
BatchTrainPatterns( patternsselected, rate);
|
||||
}
|
||||
// turn all "selected" flags off again
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
patterns[i].selected = false;
|
||||
}
|
||||
}
|
||||
|
||||
// trains the net using batch training
|
||||
// takes a number of patterns
|
||||
public void BatchTrainPatterns(Pattern[] patterns, double rate) {
|
||||
for (int i = 0; i < patterns.length; i++) {
|
||||
BatchTrain(rate, patterns[i]);
|
||||
}
|
||||
// update weights using cumulative values obtained during batch training
|
||||
for ( int i = 0; i < neurons.length; i++ ) {
|
||||
if(neurons[i].layer != 0) {
|
||||
neurons[i].BatchUpdateWeights(patterns.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trains the net using batch training
|
||||
// takes only one pattern
|
||||
public void BatchTrain(double rate, Pattern pattern) {
|
||||
// feed fw
|
||||
FeedForward(pattern.input);
|
||||
// train the output layer first
|
||||
for (int j = 0; j < layers[nolayers-1].neurons.length; j++) {
|
||||
layers[nolayers-1].neurons[j].OutputBatchTrain(rate, pattern.target[j]);
|
||||
}
|
||||
// train hidden layers
|
||||
for (int i = nolayers-2; i > 0; i--) {
|
||||
for (int j = 0; j < layers[i].neurons.length; j++) {
|
||||
layers[i].neurons[j].HiddenBatchTrain(rate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// represents an array of neurons belonging to the same layer.
|
||||
class Layer implements Serializable {
|
||||
Neuron[] neurons;
|
||||
// constructs a layer object
|
||||
public Layer(int layerno) {
|
||||
int counter = 0;
|
||||
// see how many neurons there are in this layer
|
||||
for (int i = 0; i < NeuralNet.this.neurons.length; i++) {
|
||||
if (NeuralNet.this.neurons[i].layer == layerno) {counter++;}
|
||||
}
|
||||
// create an array of neurons
|
||||
this.neurons = new Neuron[counter];
|
||||
// place neurons
|
||||
counter = 0;
|
||||
for (int i = 0; i < NeuralNet.this.neurons.length; i++) {
|
||||
if (NeuralNet.this.neurons[i].layer == layerno) {
|
||||
this.neurons[counter++] = NeuralNet.this.neurons[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class Neuron implements Serializable {
|
||||
|
||||
public int id; // to be used in saveconfig method
|
||||
public double threshold;
|
||||
private double prevthreshold;
|
||||
public int layer;
|
||||
public double output;
|
||||
public char axonfamily; // logistic? hyperbolic tangent? linear?
|
||||
protected double momentumrate;
|
||||
protected double axonfuncflatness; // if the axon func. is a curve like sigmoid, this indicates the flatness paramater.
|
||||
protected double learningratecoefficient; // i.e.: if the learning rate is .1 and this value is 1.5, actual learning rate will be .1 * 1.5 = .15
|
||||
public Neuron[] neuronsout; // array of neurons which take this neuron's output. To be used during back propagation
|
||||
public Neuron[] neuronsin; // array of neurons from which this neuron takes outputs. To be used during feedforward
|
||||
public Synapse[] synapsesout; // array of synapses which take this neuron's output. To be used during back propagation
|
||||
public Synapse[] synapsesin; // array of synapses from which this neuron takes outputs. To be used during feedforward
|
||||
protected double error; // to be used during bp.
|
||||
protected double cumulthresholddiff; // cumulate changes in threshold here during batch training
|
||||
|
||||
// constructor for input neurons
|
||||
public Neuron (int id) {
|
||||
this.id = id;
|
||||
this.layer = 0;
|
||||
}
|
||||
|
||||
// another constructor
|
||||
public Neuron (int id, int layer, double axonfuncflatness, char axonfamily, double momentumrate, double learningratecoefficient, Randomizer randomizer) {
|
||||
output = 0;
|
||||
this.axonfamily = axonfamily;
|
||||
threshold = randomizer.Uniform(-1,1);
|
||||
prevthreshold = threshold;
|
||||
this.id = id;
|
||||
this.layer = layer;
|
||||
this.momentumrate = momentumrate;
|
||||
this.axonfuncflatness = axonfuncflatness;
|
||||
this.learningratecoefficient = learningratecoefficient;
|
||||
cumulthresholddiff = 0;
|
||||
}
|
||||
|
||||
// this method constructs neuronin and neuronout arrays in order to determine the relationships of this neuron with others.
|
||||
// should be called during the construction of the net
|
||||
public void InsOuts (Neuron[] neuronsin, Neuron[] neuronsout, Synapse[] synapsesin, Synapse[] synapsesout) {
|
||||
|
||||
this.neuronsin = neuronsin;
|
||||
this.neuronsout = neuronsout;
|
||||
this.synapsesin = synapsesin;
|
||||
this.synapsesout = synapsesout;
|
||||
}
|
||||
|
||||
// updates the output and the activation according to the inputs
|
||||
public void UpdateOutput () {
|
||||
// first sum inputs and find the activation
|
||||
double activation = 0;
|
||||
for (int i = 0; i < neuronsin.length; i++) {
|
||||
activation += neuronsin[i].output * synapsesin[i].weight;
|
||||
}
|
||||
activation += -1 * threshold;
|
||||
// calculate the output using the activation function of this neuron
|
||||
switch (axonfamily) {
|
||||
case 'g': // logistic
|
||||
output = 1 / ( 1 + Math.exp( - activation / axonfuncflatness ) );
|
||||
break;
|
||||
case 't': // hyperbolic tangent (tanh)
|
||||
output = ( 2 / ( 1 + Math.exp( - activation / axonfuncflatness ) ) ) - 1;
|
||||
/* // alternatively,
|
||||
double temp = Math.exp( 2 * ( activation / axonfuncflatness ) ) ; // so that the computation is faster
|
||||
output = ( temp - 1 ) / ( temp + 1 );
|
||||
*/
|
||||
break;
|
||||
case 'l': // linear
|
||||
output = activation;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Incremantal train ------------------------------------------
|
||||
|
||||
// trains the output neurons using incremental training
|
||||
public void OutputIncrementalTrain (double rate, double target) {
|
||||
this.error = (target - output) * Derivative();
|
||||
IncrementalUpdateWeights(rate);
|
||||
}
|
||||
|
||||
// trains the hidden neurons using incremental training
|
||||
public void HiddenIncrementalTrain (double rate) {
|
||||
// first compute the error
|
||||
double temp_diff = 0;
|
||||
for (int i = 0; i < neuronsout.length; i++) {
|
||||
temp_diff += neuronsout[i].error * synapsesout[i].prevweight;
|
||||
}
|
||||
error = temp_diff * Derivative();
|
||||
IncrementalUpdateWeights(rate);
|
||||
}
|
||||
|
||||
// updates weights according to the error
|
||||
private void IncrementalUpdateWeights (double rate) {
|
||||
double temp_weight;
|
||||
for (int i = 0; i < synapsesin.length; i++) {
|
||||
temp_weight = synapsesin[i].weight;
|
||||
synapsesin[i].weight += (rate * learningratecoefficient * error * neuronsin[i].output) + ( momentumrate * ( synapsesin[i].weight - synapsesin[i].prevweight ) );
|
||||
synapsesin[i].prevweight = temp_weight;
|
||||
if (synapsesin[i].cumulweightdiff != 0) {synapsesin[i].cumulweightdiff = 0;}
|
||||
}
|
||||
temp_weight = threshold;
|
||||
threshold += ( rate * learningratecoefficient * error * -1 ) + ( momentumrate * ( threshold - prevthreshold ) );
|
||||
prevthreshold = temp_weight;
|
||||
if (cumulthresholddiff != 0) {cumulthresholddiff = 0;}
|
||||
}
|
||||
|
||||
// Batch train ------------------------------------------------
|
||||
|
||||
// trains the output neurons using batch training
|
||||
public void OutputBatchTrain (double rate, double target) {
|
||||
this.error = (target - output) * Derivative();
|
||||
BatchCumulateWeights(rate);
|
||||
}
|
||||
|
||||
// trains the hidden neurons using batch training
|
||||
public void HiddenBatchTrain (double rate) {
|
||||
// first compute the error
|
||||
double temp_diff = 0;
|
||||
for (int i = 0; i < neuronsout.length; i++) {
|
||||
temp_diff += neuronsout[i].error * synapsesout[i].weight;
|
||||
}
|
||||
error = temp_diff * Derivative();
|
||||
BatchCumulateWeights(rate);
|
||||
}
|
||||
|
||||
// cumulates weights according to the error
|
||||
private void BatchCumulateWeights (double rate) {
|
||||
double temp_diff;
|
||||
for (int i = 0; i < synapsesin.length; i++) {
|
||||
synapsesin[i].cumulweightdiff += rate * learningratecoefficient * error * neuronsin[i].output;
|
||||
}
|
||||
cumulthresholddiff += rate * learningratecoefficient * error * -1;
|
||||
}
|
||||
|
||||
// updates weights according to the cumulated weights
|
||||
public void BatchUpdateWeights (int noofepochs) {
|
||||
double temp_weight;
|
||||
for (int i = 0; i < synapsesin.length; i++) {
|
||||
temp_weight = synapsesin[i].weight;
|
||||
synapsesin[i].weight += ( synapsesin[i].cumulweightdiff / noofepochs ) + ( momentumrate * ( synapsesin[i].weight - synapsesin[i].prevweight ) );
|
||||
synapsesin[i].prevweight = temp_weight;
|
||||
synapsesin[i].cumulweightdiff = 0;
|
||||
}
|
||||
temp_weight = threshold;
|
||||
threshold += ( cumulthresholddiff / noofepochs ) + ( momentumrate * ( threshold - prevthreshold ) );
|
||||
prevthreshold = temp_weight;
|
||||
cumulthresholddiff = 0;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// returns the value of the derivative of the activation function for the last activation value
|
||||
public double Derivative () {
|
||||
double temp_derivative;
|
||||
switch (axonfamily) {
|
||||
case 'g': // logistic
|
||||
temp_derivative = ( output * ( 1 - output ) ) / axonfuncflatness; break;
|
||||
case 't': // hyperbolic tangent
|
||||
temp_derivative = ( 1 - Math.pow( output , 2 ) ) / ( 2 * axonfuncflatness ); break;
|
||||
// temp_derivative = Math.pow( ( 2 / ( Math.exp(activation / axonfuncflatness ) + Math.exp( - activation / axonfuncflatness ) ) ) ,2 ) / axonfuncflatness; break;
|
||||
case 'l': // linear
|
||||
temp_derivative = 1; break;
|
||||
default: temp_derivative = 0; break;
|
||||
}
|
||||
return temp_derivative;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
// represents a pattern
|
||||
public class Pattern {
|
||||
public double[] input;
|
||||
public double[] target;
|
||||
public boolean selected;
|
||||
// constructor
|
||||
public Pattern (double[] temp_inputs, double[] temp_targets) {
|
||||
input = new double[temp_inputs.length];
|
||||
target = new double[temp_targets.length];
|
||||
for ( int i = 0; i < temp_inputs.length; i++) {
|
||||
input[i] = temp_inputs[i];
|
||||
}
|
||||
for ( int i = 0; i < temp_targets.length; i++) {
|
||||
target[i] = temp_targets[i];
|
||||
}
|
||||
selected = false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
|
||||
// represents all patterns which will be used during training
|
||||
class PatternSet {
|
||||
|
||||
Pattern[] patterns; // whole pattern set including all patterns
|
||||
Pattern[] trainingpatterns; // patterns to be used during training
|
||||
Pattern[] crossvalpatterns; // patterns to be used during cross validation
|
||||
Pattern[] testpatterns; // patterns to be used using testing
|
||||
double[] crossvaldeviations;
|
||||
double[] testdeviations;
|
||||
private Randomizer randomizer;
|
||||
|
||||
// constructor.
|
||||
// parameters should sum to 1
|
||||
public PatternSet (String sourceFile, int noofinputs, int nooftargets, double ratiotraining, double ratiocrossval, double ratiotest, Randomizer randomizer) {
|
||||
|
||||
// load patterns from the file
|
||||
|
||||
// 1st determine how many patterns there are
|
||||
LineReader linereader = new LineReader(sourceFile);
|
||||
int counter = 0;
|
||||
double temp_double;
|
||||
while (linereader.NextLineSplitted()){
|
||||
try {
|
||||
// count numeric data only
|
||||
for ( int i = 0; i < (noofinputs + nooftargets); i++) {
|
||||
temp_double = Double.parseDouble(linereader.column[i]);
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
catch (NumberFormatException e) {}
|
||||
}
|
||||
linereader = null;
|
||||
patterns = new Pattern[counter];
|
||||
|
||||
// then read each pattern and place it into the array
|
||||
double[] temp_in = new double[noofinputs];
|
||||
double[] temp_tar = new double[nooftargets];
|
||||
linereader = new LineReader(sourceFile);
|
||||
counter = 0;
|
||||
while (linereader.NextLineSplitted()){
|
||||
try { // count numeric data only
|
||||
for (int i = 0; i < noofinputs; i++) {
|
||||
temp_in[i] = Double.parseDouble(linereader.column[i]);
|
||||
}
|
||||
for (int i = noofinputs; i < noofinputs+nooftargets; i++) {
|
||||
temp_tar[i-noofinputs] = Double.parseDouble(linereader.column[i]);
|
||||
}
|
||||
patterns[counter++] = new Pattern(temp_in, temp_tar);
|
||||
}
|
||||
catch (NumberFormatException e) {}
|
||||
}
|
||||
linereader = null;
|
||||
|
||||
// now determine the no. of training, cross val. and test patterns
|
||||
trainingpatterns = new Pattern[(int)(Math.round(patterns.length * ratiotraining))];
|
||||
crossvalpatterns = new Pattern[(int)(Math.round(patterns.length * ratiocrossval))];
|
||||
testpatterns = new Pattern[patterns.length - trainingpatterns.length - crossvalpatterns.length];
|
||||
|
||||
int patterntoselect;
|
||||
int patternsnotselected = patterns.length;
|
||||
|
||||
// first create training patterns
|
||||
for ( int i = 0; i < trainingpatterns.length; i++ ) {
|
||||
patterntoselect = randomizer.random.nextInt(patternsnotselected);
|
||||
counter = 0;
|
||||
for ( int j = 0; j < patterns.length; j++ ) {
|
||||
if ( !patterns[j].selected ) {
|
||||
if ( counter == patterntoselect ) {
|
||||
trainingpatterns[i] = patterns[j];
|
||||
patterns[j].selected = true;
|
||||
patternsnotselected--;
|
||||
break;
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// then create cross validation patterns
|
||||
for ( int i = 0; i < crossvalpatterns.length; i++ ) {
|
||||
patterntoselect = randomizer.random.nextInt(patternsnotselected);
|
||||
counter = 0;
|
||||
for ( int j = 0; j < patterns.length; j++ ) {
|
||||
if ( !patterns[j].selected ) {
|
||||
if ( counter == patterntoselect ) {
|
||||
crossvalpatterns[i] = patterns[j];
|
||||
patterns[j].selected = true;
|
||||
patternsnotselected--;
|
||||
break;
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// and then create test patterns
|
||||
for ( int i = 0; i < testpatterns.length; i++ ) {
|
||||
patterntoselect = randomizer.random.nextInt(patternsnotselected);
|
||||
counter = 0;
|
||||
for ( int j = 0; j < patterns.length; j++ ) {
|
||||
if ( !patterns[j].selected ) {
|
||||
if ( counter == patterntoselect ) {
|
||||
testpatterns[i] = patterns[j];
|
||||
patterns[j].selected = true;
|
||||
patternsnotselected--;
|
||||
break;
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// and now switch all patterns as !selected, so that they are ready for training
|
||||
for ( int i = 0; i < patterns.length; i++ ) {
|
||||
patterns[i].selected = false;
|
||||
}
|
||||
|
||||
// calculate average deviations for cross val data as well as test data
|
||||
double[] averages = new double[nooftargets];
|
||||
crossvaldeviations = new double[nooftargets];
|
||||
testdeviations = new double[nooftargets];
|
||||
for (int i = 0; i < nooftargets; i++) {
|
||||
// first calculate crossval deviations
|
||||
averages[i] = 0;
|
||||
for (int j = 0; j < crossvalpatterns.length; j++) {
|
||||
averages[i] += crossvalpatterns[j].target[i];
|
||||
}
|
||||
averages[i] /= crossvalpatterns.length;
|
||||
// now calculate deviations
|
||||
crossvaldeviations[i] = 0;
|
||||
for (int j = 0; j < crossvalpatterns.length; j++) {
|
||||
crossvaldeviations[i] += Math.abs(crossvalpatterns[j].target[i] - averages[i]);
|
||||
}
|
||||
crossvaldeviations[i] = crossvaldeviations[i] * 2 / crossvalpatterns.length;
|
||||
|
||||
// then calculate test deviations
|
||||
averages[i] = 0;
|
||||
for (int j = 0; j < testpatterns.length; j++) {
|
||||
averages[i] += testpatterns[j].target[i];
|
||||
}
|
||||
averages[i] /= testpatterns.length;
|
||||
// now calculate deviations
|
||||
testdeviations[i] = 0;
|
||||
for (int j = 0; j < testpatterns.length; j++) {
|
||||
testdeviations[i] += Math.abs(testpatterns[j].target[i] - averages[i]);
|
||||
}
|
||||
testdeviations[i] = testdeviations[i] * 2 / testpatterns.length;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Random;
|
||||
|
||||
public class Randomizer implements Serializable {
|
||||
|
||||
public Random random;
|
||||
|
||||
// constructor using system clock
|
||||
public Randomizer () {
|
||||
random = new Random();
|
||||
}
|
||||
|
||||
// constructor using seed
|
||||
public Randomizer ( long seed ) {
|
||||
random = new Random(seed);
|
||||
}
|
||||
|
||||
// Method to generate a uniformly distributed value between two values
|
||||
public double Uniform (double min, double max) { // throws ...
|
||||
return ( random.nextDouble() * (max - min) ) + min;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
public class Synapse implements Serializable
|
||||
{
|
||||
public double weight;
|
||||
public double prevweight; // to be used during bp
|
||||
public double cumulweightdiff; // cumulate changes in weight here during batch training
|
||||
public Neuron sourceunit;
|
||||
public Neuron targetunit;
|
||||
|
||||
// constructor
|
||||
public Synapse (Neuron sourceunit, Neuron targetunit, Randomizer randomizer)
|
||||
{
|
||||
this.sourceunit = sourceunit;
|
||||
this.targetunit = targetunit;
|
||||
cumulweightdiff = 0;
|
||||
weight = randomizer.Uniform(-1,1);
|
||||
prevweight = weight;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
public class example3 {
|
||||
public static void main(String args[]) {
|
||||
|
||||
Randomizer randomizer = new Randomizer();
|
||||
|
||||
// create a multilayer perceptron with three layers:
|
||||
// one input layer with two units; one hidden layer
|
||||
// with three neurons, using tanh function;
|
||||
// one output layer with one neuron using linear function.
|
||||
// except for noofneurons, all parameters for the input layer
|
||||
// are ineffectual.
|
||||
|
||||
int[] noofneurons = {2,3,1};
|
||||
|
||||
double[] learnratecoeff = {1, 1, 1};
|
||||
char[] axonfamily = {'t', 't', 'l'};
|
||||
double[] momentumrate = {0, .4, .4};
|
||||
double[] flatness = {1, 1, 1};
|
||||
|
||||
System.out.println("Creating the net");
|
||||
|
||||
NeuralNet mynet = new NeuralNet(noofneurons, learnratecoeff, axonfamily, momentumrate, flatness, randomizer);
|
||||
|
||||
// train the net using incremental training
|
||||
System.out.println("Beginning incremental training");
|
||||
Pattern pattern;
|
||||
double[] inputs = new double[2];
|
||||
double[] outputs = new double[1];
|
||||
for (int i = 0; i < 3000; i++) {
|
||||
inputs[0] = randomizer.Uniform(-1,1);
|
||||
inputs[1] = randomizer.Uniform(-1,1);
|
||||
outputs[0] = Math.sin(inputs[0] + inputs[1]);
|
||||
pattern = new Pattern(inputs, outputs);
|
||||
mynet.IncrementalTrain(.2, pattern);
|
||||
}
|
||||
System.out.println("Training is over");
|
||||
|
||||
// test it
|
||||
inputs[0] = -0.5;
|
||||
inputs[1] = 0.3;
|
||||
System.out.println("Feeding the net");
|
||||
outputs = mynet.Output(inputs);
|
||||
System.out.println("Sin ( -0.5 + 0.3 ) = "+outputs[0]);
|
||||
|
||||
mynet = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -232,6 +232,8 @@ public class LocalSplitGenerator implements Generator {
|
|||
// INITIALIZATION
|
||||
long tstart = System.currentTimeMillis();
|
||||
try {
|
||||
AnalysisLogger.getLogger().trace("generate->Using algorithm "+distributionModel.getName());
|
||||
|
||||
AnalysisLogger.getLogger().trace("generate->Check for table creation");
|
||||
createTable();
|
||||
AnalysisLogger.getLogger().trace("generate->Take area reference");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
package org.gcube.dataanalysis.ecoengine.processing.factories;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Model;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Modeler;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
|
||||
public class EvaluatorsFactory {
|
||||
|
||||
|
||||
public static Evaluator getEvaluator(AlgorithmConfiguration config) throws Exception {
|
||||
Evaluator evaler = (Evaluator) ProcessorsFactory.getProcessor(config, config.getConfigPath() + AlgorithmConfiguration.evaluatorsFile);
|
||||
return evaler;
|
||||
}
|
||||
|
||||
public static List<String> getAllEvaluators(String configPath) throws Exception{
|
||||
List<String> evaluators = ProcessorsFactory.getClasses(configPath + AlgorithmConfiguration.evaluatorsFile);
|
||||
return evaluators;
|
||||
}
|
||||
|
||||
public static HashMap<String,VarCouple> getEvaluatorParameters(String configPath, String algorithmName) throws Exception{
|
||||
HashMap<String,VarCouple> inputs = ProcessorsFactory.getParameters(configPath + AlgorithmConfiguration.evaluatorsFile, algorithmName);
|
||||
return inputs;
|
||||
}
|
||||
|
||||
public static List<Evaluator> getEvaluators(AlgorithmConfiguration config) throws Exception {
|
||||
List<Evaluator> evaluators = new ArrayList<Evaluator>();
|
||||
evaluators.add(getEvaluator(config));
|
||||
return evaluators;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ import java.util.Properties;
|
|||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Model;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Modeler;
|
||||
|
|
@ -16,76 +17,80 @@ import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
|||
|
||||
public class ProcessorsFactory {
|
||||
|
||||
public static HashMap<String,VarCouple> getDefaultDatabaseConfiguration(String cfgPath){
|
||||
String databasecfgfile =cfgPath+AlgorithmConfiguration.defaultConnectionFile;
|
||||
try{
|
||||
return DatabaseFactory.getDefaultDatabaseConfiguration(databasecfgfile);
|
||||
}catch(Exception e){
|
||||
public static HashMap<String, VarCouple> getDefaultDatabaseConfiguration(String cfgPath) {
|
||||
String databasecfgfile = cfgPath + AlgorithmConfiguration.defaultConnectionFile;
|
||||
try {
|
||||
return DatabaseFactory.getDefaultDatabaseConfiguration(databasecfgfile);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static List<String> getClasses(String file) throws Exception{
|
||||
|
||||
|
||||
public static List<String> getClasses(String file) throws Exception {
|
||||
|
||||
Properties p = AlgorithmConfiguration.getProperties(file);
|
||||
List<String> algs = new ArrayList<String>();
|
||||
for (Object algName: p.keySet()){
|
||||
algs.add((String)algName);
|
||||
for (Object algName : p.keySet()) {
|
||||
algs.add((String) algName);
|
||||
}
|
||||
return algs;
|
||||
}
|
||||
|
||||
|
||||
public static HashMap<String,VarCouple> getParameters(String file, String algorithmName) throws Exception{
|
||||
|
||||
|
||||
public static HashMap<String, VarCouple> getParameters(String file, String algorithmName) throws Exception {
|
||||
|
||||
Properties p = AlgorithmConfiguration.getProperties(file);
|
||||
String algorithmclass = p.getProperty(algorithmName);
|
||||
Object algclass = Class.forName(algorithmclass).newInstance();
|
||||
// if the algorithm is a generator itself then take it
|
||||
if (algclass instanceof Generator) {
|
||||
return ((Generator) algclass).getInputParameters();
|
||||
} else if (algclass instanceof Modeler) {
|
||||
return ((Modeler) algclass).getInputParameters();
|
||||
} else if (algclass instanceof Model) {
|
||||
return ((Model) algclass).getInputParameters();
|
||||
}
|
||||
else if (algclass instanceof Evaluator) {
|
||||
return ((Evaluator) algclass).getInputParameters();
|
||||
}
|
||||
else
|
||||
return ((SpatialProbabilityDistribution) algclass).getInputParameters();
|
||||
|
||||
}
|
||||
|
||||
public static Object getProcessor(AlgorithmConfiguration config, String file) throws Exception {
|
||||
// modify this class in order to take the right generator algorithm
|
||||
try {
|
||||
// initialize the logger
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
// take the algorithm
|
||||
String algorithm = config.getAgent();
|
||||
if (algorithm == null)
|
||||
throw new Exception("PROCESSOR NOT SPECIFIED");
|
||||
// take the algorithms list
|
||||
Properties p = AlgorithmConfiguration.getProperties(file);
|
||||
String algorithmclass = p.getProperty(algorithmName);
|
||||
String algorithmclass = p.getProperty(algorithm);
|
||||
Object algclass = Class.forName(algorithmclass).newInstance();
|
||||
//if the algorithm is a generator itself then take it
|
||||
if (algclass instanceof Generator){
|
||||
return ((Generator) algclass).getInputParameters();
|
||||
}
|
||||
else if (algclass instanceof Modeler){
|
||||
return ((Modeler) algclass).getInputParameters();
|
||||
}
|
||||
else if (algclass instanceof Model){
|
||||
return ((Model) algclass).getInputParameters();
|
||||
}
|
||||
if (algclass instanceof Generator) {
|
||||
Generator g = (Generator) algclass;
|
||||
g.setConfiguration(config);
|
||||
g.init();
|
||||
return g;
|
||||
} else if (algclass instanceof Modeler) {
|
||||
Modeler m = (Modeler) algclass;
|
||||
return m;
|
||||
}
|
||||
else if (algclass instanceof Evaluator) {
|
||||
Evaluator m = (Evaluator) algclass;
|
||||
return m;
|
||||
}
|
||||
else
|
||||
return ((SpatialProbabilityDistribution) algclass).getInputParameters();
|
||||
|
||||
return null;
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw e;
|
||||
}
|
||||
|
||||
public static Object getProcessor(AlgorithmConfiguration config,String file) throws Exception {
|
||||
//modify this class in order to take the right generator algorithm
|
||||
try {
|
||||
//initialize the logger
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
//take the algorithm
|
||||
String algorithm = config.getGenerator();
|
||||
if (algorithm == null) throw new Exception("PROCESSOR NOT SPECIFIED");
|
||||
//take the algorithms list
|
||||
Properties p = AlgorithmConfiguration.getProperties(file);
|
||||
String algorithmclass = p.getProperty(algorithm);
|
||||
Object algclass = Class.forName(algorithmclass).newInstance();
|
||||
if (algclass instanceof Generator){
|
||||
Generator g = (Generator) algclass;
|
||||
g.setConfiguration(config);
|
||||
g.init();
|
||||
return g;
|
||||
}
|
||||
else if (algclass instanceof Modeler){
|
||||
Modeler m = (Modeler) algclass;
|
||||
return m;
|
||||
}
|
||||
else return null;
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@ package org.gcube.dataanalysis.ecoengine.spatialdistributions;
|
|||
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
|
||||
public class AquamapsAlgorithmCore {
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
package org.gcube.dataanalysis.ecoengine.spatialdistributions;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class AquamapsNN extends AquamapsNative{
|
||||
|
||||
private Neural_Network neuralnet;
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "AQUAMAPS_NATIVE_NEURAL_NETWORK";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Aquamaps Native Algorithm calculated by neural network";
|
||||
}
|
||||
|
||||
@Override
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
|
||||
parameters.put("EnvelopeTable", new VarCouple(VARTYPE.STRING,"hspen"));
|
||||
parameters.put("PreprocessedTable", new VarCouple(VARTYPE.STRING,"maxminlat_hspen"));
|
||||
|
||||
parameters.put("SpeciesTable", new VarCouple(VARTYPE.STRING, "hspen"));
|
||||
parameters.put("CsquarecodesTable", new VarCouple(VARTYPE.STRING, "hcaf_d"));
|
||||
parameters.put("DistributionTable", new VarCouple(VARTYPE.RANDOM, "hspec_nn_"));
|
||||
parameters.put("SpeciesName", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("CreateTable", new VarCouple(VARTYPE.STRING,"true"));
|
||||
parameters.put("UserName", new VarCouple(VARTYPE.STRING, ""));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration config, SessionFactory dbHibConnection) {
|
||||
super.init(config,dbHibConnection);
|
||||
String persistencePath = config.getPersistencePath();
|
||||
String filename = persistencePath + "neuralnetwork_" + config.getParam("SpeciesName") + "_" + config.getParam("UserName");
|
||||
neuralnet = loadNN(filename);
|
||||
}
|
||||
|
||||
@Override
|
||||
public float calcProb(Object mainInfo, Object area) {
|
||||
String species = getMainInfoID(mainInfo);
|
||||
String csquarecode = (String) ((Object[]) area)[0];
|
||||
Object[] wholevector = (Object[]) area;
|
||||
Object[] inputvector = new Object[wholevector.length - 6];
|
||||
for (int i = 0; i < inputvector.length; i++) {
|
||||
inputvector[i] = wholevector[i + 1];
|
||||
// AnalysisLogger.getLogger().debug(i+": "+inputvector[i]);
|
||||
}
|
||||
// AnalysisLogger.getLogger().debug("species vs csquare:" + species + " , " + csquarecode);
|
||||
float probability = 0;
|
||||
|
||||
// if (csquarecode.equals("1000:102:2"))
|
||||
probability = propagate(inputvector);
|
||||
|
||||
return probability;
|
||||
}
|
||||
|
||||
private synchronized float propagate(Object[] inputvector) {
|
||||
double[] output = new double[1];
|
||||
|
||||
try {
|
||||
output = neuralnet.propagate(Neural_Network.preprocessObjects(inputvector));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
// double [] output = new double[1];
|
||||
float probability = (float) output[0];
|
||||
|
||||
if (probability>0.1)
|
||||
AnalysisLogger.getLogger().debug(" Probability " + probability);
|
||||
|
||||
// System.exit(0);
|
||||
return probability;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getInternalStatus() {
|
||||
return 100;
|
||||
}
|
||||
|
||||
public static synchronized Neural_Network loadNN(String nomeFile) {
|
||||
|
||||
Neural_Network nn = null;
|
||||
FileInputStream stream = null;
|
||||
try {
|
||||
stream = new FileInputStream(nomeFile);
|
||||
ObjectInputStream ois = new ObjectInputStream(stream);
|
||||
nn = (Neural_Network) ois.readObject();
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " .");
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
|
||||
return nn;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
package org.gcube.dataanalysis.ecoengine.spatialdistributions;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.neurosolutions.NeuralNet;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class AquamapsNNNS extends AquamapsNative{
|
||||
|
||||
private NeuralNet neuralnet;
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "AQUAMAPS_NEURAL_NETWORK_NS";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Aquamaps Algorithm calculated by neural network";
|
||||
}
|
||||
|
||||
@Override
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
|
||||
parameters.put("EnvelopeTable", new VarCouple(VARTYPE.STRING,"hspen"));
|
||||
parameters.put("PreprocessedTable", new VarCouple(VARTYPE.STRING,"maxminlat_hspen"));
|
||||
|
||||
parameters.put("SpeciesTable", new VarCouple(VARTYPE.STRING, "hspen"));
|
||||
parameters.put("CsquarecodesTable", new VarCouple(VARTYPE.STRING, "hcaf_d"));
|
||||
parameters.put("DistributionTable", new VarCouple(VARTYPE.RANDOM, "hspec_nn_"));
|
||||
parameters.put("SpeciesName", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("CreateTable", new VarCouple(VARTYPE.STRING,"true"));
|
||||
parameters.put("UserName", new VarCouple(VARTYPE.STRING, ""));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration config, SessionFactory dbHibConnection) {
|
||||
super.init(config,dbHibConnection);
|
||||
String persistencePath = config.getPersistencePath();
|
||||
String filename = persistencePath + "neuralnetwork_" + config.getParam("SpeciesName") + "_" + config.getParam("UserName");
|
||||
neuralnet = loadNN(filename);
|
||||
}
|
||||
|
||||
@Override
|
||||
public float calcProb(Object mainInfo, Object area) {
|
||||
String species = getMainInfoID(mainInfo);
|
||||
String csquarecode = (String) ((Object[]) area)[0];
|
||||
Object[] wholevector = (Object[]) area;
|
||||
Object[] inputvector = new Object[wholevector.length - 6];
|
||||
for (int i = 0; i < inputvector.length; i++) {
|
||||
inputvector[i] = wholevector[i + 1];
|
||||
// AnalysisLogger.getLogger().debug(i+": "+inputvector[i]);
|
||||
}
|
||||
// AnalysisLogger.getLogger().debug("species vs csquare:" + species + " , " + csquarecode);
|
||||
float probability = 0;
|
||||
|
||||
// if (csquarecode.equals("1000:102:2"))
|
||||
probability = propagate(inputvector);
|
||||
|
||||
return probability;
|
||||
}
|
||||
|
||||
private synchronized float propagate(Object[] inputvector) {
|
||||
double[] output = new double[1];
|
||||
|
||||
try {
|
||||
|
||||
output = neuralnet.Output(NeuralNet.preprocessObjects(inputvector));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
// double [] output = new double[1];
|
||||
float probability = (((float) output[0])+1f)/2f ;
|
||||
|
||||
if (probability>0.1)
|
||||
// if (probability<0)
|
||||
AnalysisLogger.getLogger().debug(" Probability " + probability);
|
||||
|
||||
// System.exit(0);
|
||||
return probability;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getInternalStatus() {
|
||||
return 100;
|
||||
}
|
||||
|
||||
public static synchronized NeuralNet loadNN(String nomeFile) {
|
||||
|
||||
NeuralNet nn = null;
|
||||
FileInputStream stream = null;
|
||||
try {
|
||||
stream = new FileInputStream(nomeFile);
|
||||
ObjectInputStream ois = new ObjectInputStream(stream);
|
||||
nn = (NeuralNet) ois.readObject();
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " .");
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
|
||||
return nn;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
package org.gcube.dataanalysis.ecoengine.spatialdistributions;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VARTYPE;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.VarCouple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class AquamapsNNSuitable extends AquamapsSuitable{
|
||||
|
||||
private Neural_Network neuralnet;
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "AQUAMAPS_SUITABLE_NEURAL_NETWORK";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "Aquamaps Algorithm for Suitable Environment calculated by neural network";
|
||||
}
|
||||
|
||||
@Override
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
|
||||
parameters.put("EnvelopeTable", new VarCouple(VARTYPE.STRING,"hspen"));
|
||||
parameters.put("PreprocessedTable", new VarCouple(VARTYPE.STRING,"maxminlat_hspen"));
|
||||
|
||||
parameters.put("SpeciesTable", new VarCouple(VARTYPE.STRING, "hspen"));
|
||||
parameters.put("CsquarecodesTable", new VarCouple(VARTYPE.STRING, "hcaf_d"));
|
||||
parameters.put("DistributionTable", new VarCouple(VARTYPE.RANDOM, "hspec_nn_"));
|
||||
parameters.put("SpeciesName", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("CreateTable", new VarCouple(VARTYPE.STRING,"true"));
|
||||
parameters.put("UserName", new VarCouple(VARTYPE.STRING, ""));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(AlgorithmConfiguration config, SessionFactory dbHibConnection) {
|
||||
super.init(config,dbHibConnection);
|
||||
String persistencePath = config.getPersistencePath();
|
||||
String filename = persistencePath + "neuralnetwork_" + config.getParam("SpeciesName") + "_" + config.getParam("UserName");
|
||||
neuralnet = loadNN(filename);
|
||||
}
|
||||
|
||||
@Override
|
||||
public float calcProb(Object mainInfo, Object area) {
|
||||
String species = getMainInfoID(mainInfo);
|
||||
String csquarecode = (String) ((Object[]) area)[0];
|
||||
Object[] wholevector = (Object[]) area;
|
||||
Object[] inputvector = new Object[wholevector.length - 6];
|
||||
for (int i = 0; i < inputvector.length; i++) {
|
||||
inputvector[i] = wholevector[i + 1];
|
||||
// AnalysisLogger.getLogger().debug(i+": "+inputvector[i]);
|
||||
}
|
||||
// AnalysisLogger.getLogger().debug("species vs csquare:" + species + " , " + csquarecode);
|
||||
float probability = 0;
|
||||
|
||||
// if (csquarecode.equals("1000:102:2"))
|
||||
probability = propagate(inputvector);
|
||||
|
||||
return probability;
|
||||
}
|
||||
|
||||
private synchronized float propagate(Object[] inputvector) {
|
||||
double[] output = new double[1];
|
||||
|
||||
try {
|
||||
output = neuralnet.propagate(Neural_Network.preprocessObjects(inputvector));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
// double [] output = new double[1];
|
||||
float probability = (float) output[0];
|
||||
|
||||
if (probability>0.1)
|
||||
AnalysisLogger.getLogger().debug(" Probability " + probability);
|
||||
|
||||
// System.exit(0);
|
||||
return probability;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getInternalStatus() {
|
||||
return 100;
|
||||
}
|
||||
|
||||
public static synchronized Neural_Network loadNN(String nomeFile) {
|
||||
|
||||
Neural_Network nn = null;
|
||||
FileInputStream stream = null;
|
||||
try {
|
||||
stream = new FileInputStream(nomeFile);
|
||||
ObjectInputStream ois = new ObjectInputStream(stream);
|
||||
nn = (Neural_Network) ois.readObject();
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
AnalysisLogger.getLogger().debug("Error in reading the object from file " + nomeFile + " .");
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
|
||||
return nn;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -210,7 +210,7 @@ public class AquamapsSuitable implements SpatialProbabilityDistributionTable{
|
|||
parameters.put("CsquarecodesTable", new VarCouple(VARTYPE.STRING,"hcaf_d"));
|
||||
parameters.put("DistributionTable", new VarCouple(VARTYPE.RANDOM,"hspec_"));
|
||||
parameters.put("PreprocessedTable", new VarCouple(VARTYPE.STRING,"maxminlat_hspen"));
|
||||
parameters.put("CreateTable", new VarCouple(VARTYPE.STRING,"hspen_minmaxlat"));
|
||||
parameters.put("CreateTable", new VarCouple(VARTYPE.STRING,"true"));
|
||||
parameters.put("DatabaseUserName",new VarCouple(VARTYPE.DATABASEUSERNAME,""));
|
||||
parameters.put("DatabasePassword",new VarCouple(VARTYPE.DATABASEPASSWORD,""));
|
||||
parameters.put("DatabaseURL",new VarCouple(VARTYPE.DATABASEURL,""));
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import java.util.List;
|
|||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.EvaluatorsFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.ProcessorsFactory;
|
||||
|
|
@ -50,6 +52,21 @@ public static void main(String[] args) throws Exception {
|
|||
map = ProcessorsFactory.getDefaultDatabaseConfiguration("./cfg/");
|
||||
System.out.println("Database Default Values: "+map);
|
||||
|
||||
System.out.println("\n***TEST 8- Get Database Default Parameters***");
|
||||
map = ProcessorsFactory.getDefaultDatabaseConfiguration("./cfg/");
|
||||
System.out.println("Database Default Values: "+map);
|
||||
|
||||
System.out.println("\n***TEST 9- Get Evaluators ***");
|
||||
System.out.println("Database Default Values: "+EvaluatorsFactory.getAllEvaluators("./cfg/"));
|
||||
|
||||
System.out.println("\n***TEST 10- Get Evaluator Parameters ***");
|
||||
map = EvaluatorsFactory.getEvaluatorParameters("./cfg/","QUALITY_ANALYSIS");
|
||||
System.out.println("Database Default Values: "+map);
|
||||
|
||||
System.out.println("\n***TEST 11- Get Evaluator Parameters ***");
|
||||
List<Evaluator> eval = EvaluatorsFactory.getEvaluators(testConfigEvaluator());
|
||||
System.out.println("Database Default Values: "+eval);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -94,7 +111,17 @@ public static void main(String[] args) throws Exception {
|
|||
|
||||
}
|
||||
|
||||
|
||||
private static AlgorithmConfiguration testConfigEvaluator() {
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setNumberOfResources(2);
|
||||
|
||||
config.setAgent("QUALITY_ANALYSIS");
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
private static AlgorithmConfiguration testConfig() {
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath("./cfg/");
|
||||
|
|
@ -102,7 +129,7 @@ public static void main(String[] args) throws Exception {
|
|||
config.setNumberOfResources(2);
|
||||
config.setModel("TEST");
|
||||
|
||||
config.setGenerator("SIMPLE_LOCAL");
|
||||
config.setAgent("SIMPLE_LOCAL");
|
||||
|
||||
return config;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,146 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test.neuralnetwork;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectOutputStream;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.models.cores.neuralnetworks.Neural_Network;
|
||||
|
||||
public class TestNN {
|
||||
|
||||
public static void main1(String[] args) {
|
||||
|
||||
int numberOfInputs = 2;
|
||||
int numberOfOutputs = 1;
|
||||
int[] innerLayers = Neural_Network.setupInnerLayers(2);
|
||||
|
||||
Neural_Network nn = new Neural_Network(numberOfInputs, numberOfOutputs, innerLayers, Neural_Network.ACTIVATIONFUNCTION.SIGMOID);
|
||||
|
||||
double[] input1 = { 1, 1 };
|
||||
input1 = Neural_Network.preprocessObjects(input1);
|
||||
|
||||
double[] output1 = nn.getPositiveCase();
|
||||
|
||||
double[] input2 = { 0, 0 };
|
||||
input1 = Neural_Network.preprocessObjects(input1);
|
||||
|
||||
double[] output2 = nn.getNegativeCase();
|
||||
|
||||
double[] input3 = { 1, 0 };
|
||||
input1 = Neural_Network.preprocessObjects(input1);
|
||||
|
||||
double[] output3 = nn.getNegativeCase();
|
||||
|
||||
double[] input4 = { 0, 1 };
|
||||
input1 = Neural_Network.preprocessObjects(input1);
|
||||
double[] output4 = nn.getNegativeCase();
|
||||
|
||||
double[][] in = new double[4][];
|
||||
double[][] out = new double[4][];
|
||||
in[0] = input1;
|
||||
out[0] = output1;
|
||||
in[1] = input2;
|
||||
out[1] = output2;
|
||||
in[2] = input3;
|
||||
out[2] = output3;
|
||||
in[3] = input4;
|
||||
out[3] = output4;
|
||||
|
||||
nn.train(in, out);
|
||||
|
||||
System.out.println("addestramento compiuto");
|
||||
|
||||
double[] nninput = { 1, 1 };
|
||||
Neural_Network.preprocessObjects(nninput);
|
||||
|
||||
nn.setAcceptanceThreshold(0.80f);
|
||||
|
||||
double[] nnout = nn.getClassification(nn.propagate(nninput));
|
||||
|
||||
System.out.println("calcolo compiuto");
|
||||
System.out.println();
|
||||
|
||||
for (int i = 0; i < nnout.length; i++) {
|
||||
// nn.writeout(nnout[i],0.5);
|
||||
System.out.print("valore reale: " + nnout[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
int numberOfInputs = 13;
|
||||
int numberOfOutputs = 1;
|
||||
int[] innerLayers = Neural_Network.setupInnerLayers(2);
|
||||
|
||||
Neural_Network nn = new Neural_Network(numberOfInputs, numberOfOutputs, innerLayers, Neural_Network.ACTIVATIONFUNCTION.SIGMOID);
|
||||
|
||||
double[] input1 = new double[numberOfInputs];
|
||||
for (int i = 0; i < numberOfInputs; i++) {
|
||||
input1[i] = 20 * Math.random();
|
||||
}
|
||||
input1 = Neural_Network.preprocessObjects(input1);
|
||||
double[] output1 = nn.getPositiveCase();
|
||||
|
||||
double[] input2 = new double[numberOfInputs];
|
||||
for (int i = 0; i < numberOfInputs; i++) {
|
||||
input2[i] = 20 * Math.random();
|
||||
}
|
||||
|
||||
input2 = Neural_Network.preprocessObjects(input2);
|
||||
double[] output2 = nn.getNegativeCase();
|
||||
|
||||
double[][] in = new double[2][];
|
||||
double[][] out = new double[2][];
|
||||
in[0] = input1;
|
||||
out[0] = output1;
|
||||
in[1] = input2;
|
||||
out[1] = output2;
|
||||
|
||||
nn.train(in, out);
|
||||
|
||||
System.out.println("addestramento compiuto");
|
||||
salva("./cfg/nn_Fis-22747",nn);
|
||||
|
||||
double[] inputnn = new double[numberOfInputs];
|
||||
for (int i = 0; i < numberOfInputs; i++) {
|
||||
inputnn[i] = Math.random();
|
||||
}
|
||||
|
||||
Neural_Network.preprocessObjects(inputnn);
|
||||
|
||||
nn.setAcceptanceThreshold(0.80f);
|
||||
|
||||
double[] nnout = nn.getClassification(nn.propagate(inputnn));
|
||||
|
||||
System.out.println("calcolo compiuto");
|
||||
System.out.println();
|
||||
|
||||
for (int i = 0; i < nnout.length; i++) {
|
||||
// nn.writeout(nnout[i],0.5);
|
||||
System.out.print("valore reale: " + nnout[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void salva(String nomeFile, Neural_Network nn) {
|
||||
|
||||
File f = new File(nomeFile);
|
||||
FileOutputStream stream = null;
|
||||
try {
|
||||
stream = new FileOutputStream(f);
|
||||
ObjectOutputStream oos = new ObjectOutputStream(stream);
|
||||
oos.writeObject(nn);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
System.out.println("Errore di scrittura dell'oggetto sul File: " + nomeFile);
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
System.out.println("OK scrittura dell'oggetto sul File: " + nomeFile);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test.regressions;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.evaluation.DiscrepancyAnalysis;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.EvaluatorsFactory;
|
||||
|
||||
public class TestEvaluation {
|
||||
/**
|
||||
* example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species
|
||||
*
|
||||
*/
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
//test Quality
|
||||
List<Evaluator> evaluators = EvaluatorsFactory.getEvaluators(testQuality());
|
||||
evaluate(evaluators.get(0),testQuality());
|
||||
evaluators = null;
|
||||
|
||||
System.out.println("\n**********-------************\n");
|
||||
|
||||
//test Discrepancy
|
||||
evaluators = EvaluatorsFactory.getEvaluators(testDiscrepancy());
|
||||
evaluate(evaluators.get(0),testDiscrepancy());
|
||||
evaluators = null;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static void evaluate(Evaluator evaluator, AlgorithmConfiguration config) throws Exception {
|
||||
|
||||
if (evaluator != null) {
|
||||
TestEvaluation tgs = new TestEvaluation();
|
||||
ThreadCalculator tc = tgs.new ThreadCalculator(evaluator,config);
|
||||
Thread t = new Thread(tc);
|
||||
t.start();
|
||||
while (evaluator.getStatus() < 100) {
|
||||
|
||||
String resLoad = evaluator.getResourceLoad();
|
||||
String ress = evaluator.getResources();
|
||||
|
||||
System.out.println("LOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
|
||||
System.out.println("STATUS: " + evaluator.getStatus());
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
|
||||
String resLoad = evaluator.getResourceLoad();
|
||||
String ress = evaluator.getResources();
|
||||
System.out.println("\nLOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
System.out.println("STATUS: " + evaluator.getStatus());
|
||||
|
||||
} else
|
||||
AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported");
|
||||
|
||||
}
|
||||
|
||||
public class ThreadCalculator implements Runnable {
|
||||
Evaluator dg;
|
||||
AlgorithmConfiguration config;
|
||||
|
||||
public ThreadCalculator(Evaluator dg, AlgorithmConfiguration config) {
|
||||
this.dg = dg;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
|
||||
HashMap<String, String> out = dg.process(config);
|
||||
DiscrepancyAnalysis.visualizeResults(out);
|
||||
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static AlgorithmConfiguration testQuality() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./cfg/");
|
||||
config.setNumberOfResources(1);
|
||||
config.setAgent("QUALITY_ANALYSIS");
|
||||
|
||||
config.setParam("PositiveCasesTable","presence_data_baskingshark");
|
||||
config.setParam("NegativeCasesTable","absence_data_baskingshark2");
|
||||
|
||||
config.setParam("PositiveCasesTableKeyColumn","csquarecode");
|
||||
config.setParam("NegativeCasesTableKeyColumn","csquarecode");
|
||||
|
||||
config.setParam("DistributionTable","hspec_native_baskingshark_aquamaps");
|
||||
config.setParam("DistributionTableKeyColumn","csquarecode");
|
||||
config.setParam("DistributionTableProbabilityColumn","probability");
|
||||
|
||||
config.setParam("PositiveThreshold","0.5");
|
||||
config.setParam("NegativeThreshold","0.5");
|
||||
|
||||
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static AlgorithmConfiguration testDiscrepancy() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./cfg/");
|
||||
config.setNumberOfResources(1);
|
||||
config.setAgent("DISCREPANCY_ANALYSIS");
|
||||
|
||||
config.setParam("FirstTable","hspec_native_baskingshark_aquamaps");
|
||||
config.setParam("SecondTable","hspec_suitable_nn_Fis22747");
|
||||
|
||||
config.setParam("FirstTableCsquareColumn","csquarecode");
|
||||
config.setParam("SecondTableCsquareColumn","csquarecode");
|
||||
|
||||
config.setParam("FirstTableProbabilityColumn","probability");
|
||||
config.setParam("SecondTableProbabilityColumn","probability");
|
||||
|
||||
config.setParam("ComparisonThreshold","0.1");
|
||||
|
||||
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test.regressions;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
|
||||
|
||||
public class TestSingleHSPECGeneration {
|
||||
/**
|
||||
* example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species
|
||||
*
|
||||
*/
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
List<Generator> generators = GeneratorsFactory.getGenerators(testConfigLocal());
|
||||
generators.get(0).init();
|
||||
generate(generators.get(0));
|
||||
generators = null;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static void generate(Generator generator) throws Exception {
|
||||
|
||||
if (generator != null) {
|
||||
TestSingleHSPECGeneration tgs = new TestSingleHSPECGeneration();
|
||||
ThreadCalculator tc = tgs.new ThreadCalculator(generator);
|
||||
Thread t = new Thread(tc);
|
||||
t.start();
|
||||
while (generator.getStatus() < 100) {
|
||||
|
||||
String resLoad = generator.getResourceLoad();
|
||||
String ress = generator.getResources();
|
||||
String species = generator.getLoad();
|
||||
System.out.println("LOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
System.out.println("SPECIES: " + species);
|
||||
System.out.println("STATUS: " + generator.getStatus());
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
} else
|
||||
AnalysisLogger.getLogger().trace("Generator Algorithm Not Supported");
|
||||
|
||||
}
|
||||
|
||||
public class ThreadCalculator implements Runnable {
|
||||
Generator dg;
|
||||
|
||||
public ThreadCalculator(Generator dg) {
|
||||
this.dg = dg;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
|
||||
dg.generate();
|
||||
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static AlgorithmConfiguration testConfigLocal() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setNumberOfResources(5);
|
||||
config.setModel("AQUAMAPS_SUITABLE");
|
||||
|
||||
config.setParam("DistributionTable","hspec_suitable_baskingshark_aquamaps");
|
||||
config.setParam("CsquarecodesTable","hcaf_d");
|
||||
config.setParam("EnvelopeTable","hspen_baskingshark");
|
||||
config.setParam("PreprocessedTable", "maxminlat_hspen");
|
||||
config.setParam("CreateTable","true");
|
||||
/*
|
||||
config.setParam("DatabaseUserName","gcube");
|
||||
config.setParam("DatabasePassword","d4science2");
|
||||
config.setParam("DatabaseURL","jdbc:postgresql://localhost/testdb");
|
||||
config.setParam("DatabaseDriver","org.postgresql.Driver");
|
||||
*/
|
||||
|
||||
return config;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test;
|
||||
package org.gcube.dataanalysis.ecoengine.test.regressions;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -1,10 +1,9 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test;
|
||||
package org.gcube.dataanalysis.ecoengine.test.regressions;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Modeler;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory;
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test;
|
||||
package org.gcube.dataanalysis.ecoengine.test.regressions;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test.trainers;
|
||||
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.user.EvaluatorT;
|
||||
|
||||
public class TrainProduceTestMap {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String configPath = "./cfg/";
|
||||
String presenceTable = "presence_data_baskingshark";
|
||||
String absenceTable = "absence_data_baskingshark_random";
|
||||
String csquareTable = "hcaf_d";
|
||||
String preprocessedTable = "maxminlat_hspen";
|
||||
String envelopeTable = "hspen_baskingshark";
|
||||
String commonkeycolumn = "csquarecode";
|
||||
String probabilitycolumn = "probability";
|
||||
String positiveClassificationThreshold = "0.8";
|
||||
String negativeClassificationThreshold = "0.3";
|
||||
String comparisonThreshold = "0.1";
|
||||
int numberOfResources = 4;
|
||||
String speciesCode = "Fis-22747";
|
||||
String userName = "gianpaolo.coro";
|
||||
String modelName = "AQUAMAPSNN";
|
||||
String generatorName = "AQUAMAPS_SUITABLE_NEURALNETWORK";
|
||||
String qualityOperationName = "QUALITY_ANALYSIS";
|
||||
String discrepancyOperationName = "DISCREPANCY_ANALYSIS";
|
||||
|
||||
String manualTableOfComparison = "hspec_native_baskingshark";
|
||||
String manualTableCsquareColumn = "csquare";
|
||||
String secondTableOfComparison = "hspec_native_baskingshark_aquamaps";
|
||||
|
||||
String neuralNetworkLayers= "300";
|
||||
|
||||
// String finalDistributionTable = "hspec_nn_baskingshark_close_best";
|
||||
// String finalDistributionKeyColumn= "csquarecode";
|
||||
|
||||
// String finalDistributionTable = "hspec_nn_baskingshark_random_best2";
|
||||
// String finalDistributionKeyColumn= "csquarecode";
|
||||
|
||||
// String finalDistributionTable = "hspec_native_baskingshark";
|
||||
// String finalDistributionKeyColumn= "csquare";
|
||||
|
||||
String finalDistributionTable = "hspec_native_baskingshark_aquamaps";
|
||||
String finalDistributionKeyColumn= "csquarecode";
|
||||
|
||||
//Train
|
||||
// ModelerT.train(ModelerT.getTrainingConfig(modelName, absenceTable, presenceTable, speciesCode, userName, neuralNetworkLayers, configPath));
|
||||
|
||||
//Generate
|
||||
// GeneratorT.generate(GeneratorT.getGenerationConfig(numberOfResources, generatorName, envelopeTable, preprocessedTable, speciesCode, userName, csquareTable, finalDistributionTable, configPath));
|
||||
|
||||
//Test1:Quality
|
||||
System.out.println("\nQUALITY CHECK:\n");
|
||||
EvaluatorT.evaluate(EvaluatorT.getEvaluationConfiguration(configPath, qualityOperationName, presenceTable, absenceTable, commonkeycolumn, commonkeycolumn, finalDistributionTable, finalDistributionKeyColumn, probabilitycolumn, positiveClassificationThreshold, negativeClassificationThreshold));
|
||||
|
||||
//Test2: compare to manual map
|
||||
System.out.println("\nDISCREPANCY CHECK : "+manualTableOfComparison+" vs "+finalDistributionTable+"\n");
|
||||
EvaluatorT.evaluate(EvaluatorT.getDiscrepancyConfiguration(discrepancyOperationName, manualTableOfComparison, finalDistributionTable,
|
||||
manualTableCsquareColumn, finalDistributionKeyColumn, probabilitycolumn, probabilitycolumn, comparisonThreshold, configPath));
|
||||
|
||||
//Test3: compare to automatic map
|
||||
System.out.println("\nDISCREPANCY CHECK : "+secondTableOfComparison+" vs "+finalDistributionTable+"\n");
|
||||
EvaluatorT.evaluate(EvaluatorT.getDiscrepancyConfiguration(discrepancyOperationName, secondTableOfComparison, finalDistributionTable,
|
||||
commonkeycolumn,finalDistributionKeyColumn, probabilitycolumn, probabilitycolumn, comparisonThreshold, configPath));
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
package org.gcube.dataanalysis.ecoengine.user;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.evaluation.DiscrepancyAnalysis;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Evaluator;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.EvaluatorsFactory;
|
||||
|
||||
public class EvaluatorT implements Runnable{
|
||||
|
||||
Evaluator dg;
|
||||
AlgorithmConfiguration config;
|
||||
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
|
||||
HashMap<String, String> out = dg.process(config);
|
||||
DiscrepancyAnalysis.visualizeResults(out);
|
||||
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
public EvaluatorT(Evaluator dg, AlgorithmConfiguration config) {
|
||||
this.dg = dg;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public static void evaluate(AlgorithmConfiguration config) throws Exception {
|
||||
|
||||
List<Evaluator> evaluators = EvaluatorsFactory.getEvaluators(config);
|
||||
Evaluator evaluator = evaluators.get(0);
|
||||
|
||||
if (evaluator != null) {
|
||||
EvaluatorT tc = new EvaluatorT(evaluator,config);
|
||||
Thread t = new Thread(tc);
|
||||
t.start();
|
||||
while (evaluator.getStatus() < 100) {
|
||||
|
||||
String resLoad = evaluator.getResourceLoad();
|
||||
String ress = evaluator.getResources();
|
||||
|
||||
System.out.println("LOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
|
||||
System.out.println("STATUS: " + evaluator.getStatus());
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
|
||||
String resLoad = evaluator.getResourceLoad();
|
||||
String ress = evaluator.getResources();
|
||||
System.out.println("\nLOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
System.out.println("STATUS: " + evaluator.getStatus());
|
||||
|
||||
} else
|
||||
System.out.println("Generator Algorithm Not Supported");
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static AlgorithmConfiguration getEvaluationConfiguration(String configPath,String operationName,String positiveCasesTable,String negativeCasesTable,String positiveCasesTableKeyColumn,
|
||||
String negativeCasesKeyColumn, String finalDistributionTable,String finalDistributionTableKeyColumn,String finalDistributionTableProbabilityColumn,String positiveThreshold,String negativeThreshold) {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath(configPath);
|
||||
config.setPersistencePath(configPath);
|
||||
config.setNumberOfResources(1);
|
||||
config.setAgent(operationName);
|
||||
|
||||
config.setParam("PositiveCasesTable",positiveCasesTable);
|
||||
config.setParam("NegativeCasesTable",negativeCasesTable);
|
||||
|
||||
config.setParam("PositiveCasesTableKeyColumn",positiveCasesTableKeyColumn);
|
||||
config.setParam("NegativeCasesTableKeyColumn",negativeCasesKeyColumn);
|
||||
|
||||
config.setParam("DistributionTable",finalDistributionTable);
|
||||
config.setParam("DistributionTableKeyColumn",finalDistributionTableKeyColumn);
|
||||
config.setParam("DistributionTableProbabilityColumn",finalDistributionTableProbabilityColumn);
|
||||
|
||||
config.setParam("PositiveThreshold",positiveThreshold);
|
||||
config.setParam("NegativeThreshold",negativeThreshold);
|
||||
|
||||
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
public static AlgorithmConfiguration getDiscrepancyConfiguration(String operationName, String firstTable, String secondTable, String firstTableCsquareColumn,
|
||||
String secondTableCsquareColumn, String firstTableProbabilityColumn, String secondTableProbabilityColumn, String comparisonThreshold, String configPath) {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath(configPath);
|
||||
config.setPersistencePath(configPath);
|
||||
config.setNumberOfResources(1);
|
||||
config.setAgent(operationName);
|
||||
|
||||
config.setParam("FirstTable",firstTable);
|
||||
config.setParam("SecondTable",secondTable);
|
||||
config.setParam("FirstTableCsquareColumn",firstTableCsquareColumn);
|
||||
config.setParam("SecondTableCsquareColumn",secondTableCsquareColumn);
|
||||
config.setParam("FirstTableProbabilityColumn",firstTableProbabilityColumn);
|
||||
config.setParam("SecondTableProbabilityColumn",secondTableProbabilityColumn);
|
||||
config.setParam("ComparisonThreshold",comparisonThreshold);
|
||||
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package org.gcube.dataanalysis.ecoengine.user;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Generator;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory;
|
||||
|
||||
public class GeneratorT implements Runnable{
|
||||
Generator dg;
|
||||
|
||||
public GeneratorT(Generator dg) {
|
||||
this.dg = dg;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
|
||||
dg.generate();
|
||||
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void generate(AlgorithmConfiguration config) throws Exception {
|
||||
|
||||
List<Generator> generators = GeneratorsFactory.getGenerators(config);
|
||||
Generator generator = generators.get(0);
|
||||
|
||||
if (generator != null) {
|
||||
generator.init();
|
||||
GeneratorT tgs = new GeneratorT(generator);
|
||||
Thread t = new Thread(tgs);
|
||||
t.start();
|
||||
while (generator.getStatus() < 100) {
|
||||
|
||||
String resLoad = generator.getResourceLoad();
|
||||
String ress = generator.getResources();
|
||||
String species = generator.getLoad();
|
||||
System.out.println("LOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
System.out.println("SPECIES: " + species);
|
||||
System.out.println("STATUS: " + generator.getStatus());
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
}
|
||||
else
|
||||
System.out.println("Generator Algorithm Not Supported");
|
||||
}
|
||||
|
||||
|
||||
public static AlgorithmConfiguration getGenerationConfig(int numberOfResources,String algorithmName,String envelopeTable,String preprocessedTable,String speciesName,String userName,String csquareTable,String finalDistributionTable,String configPath) {
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath(configPath);
|
||||
config.setPersistencePath(configPath);
|
||||
config.setNumberOfResources(numberOfResources);
|
||||
config.setModel(algorithmName);
|
||||
|
||||
config.setParam("EnvelopeTable", envelopeTable);
|
||||
config.setParam("PreprocessedTable", preprocessedTable);
|
||||
|
||||
config.setParam("SpeciesName", speciesName);
|
||||
config.setParam("CsquarecodesTable", csquareTable);
|
||||
config.setParam("DistributionTable", finalDistributionTable);
|
||||
config.setParam("CreateTable", "true");
|
||||
config.setParam("UserName",userName);
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package org.gcube.dataanalysis.ecoengine.user;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.Modeler;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory;
|
||||
|
||||
public class ModelerT implements Runnable{
|
||||
|
||||
Modeler dg;
|
||||
AlgorithmConfiguration config;
|
||||
|
||||
|
||||
public ModelerT(Modeler dg,AlgorithmConfiguration config) {
|
||||
this.dg = dg;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
|
||||
dg.model(config, null);
|
||||
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
public static void train(AlgorithmConfiguration config) throws Exception {
|
||||
|
||||
List<Modeler> modelers=ModelersFactory.getModelers(config);
|
||||
Modeler modeler = modelers.get(0);
|
||||
|
||||
if (modeler != null) {
|
||||
ModelerT tc = new ModelerT(modeler, config);
|
||||
Thread t = new Thread(tc);
|
||||
t.start();
|
||||
while (modeler.getStatus() < 100) {
|
||||
|
||||
String resLoad = modeler.getResourceLoad();
|
||||
String ress = modeler.getResources();
|
||||
System.out.println("LOAD: " + resLoad);
|
||||
System.out.println("RESOURCES: " + ress);
|
||||
System.out.println("STATUS: " + modeler.getStatus());
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
System.out.println("FINAL STATUS: " + modeler.getStatus());
|
||||
} else
|
||||
System.out.println("Generator Algorithm Not Supported");
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static AlgorithmConfiguration getTrainingConfig(String modelName, String absenceTable,String presenceTable,String speciesCode,String userName,String layersNeurons, String configPath){
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setConfigPath(configPath);
|
||||
config.setPersistencePath(configPath);
|
||||
config.setNumberOfResources(1);
|
||||
|
||||
config.setModel(modelName);
|
||||
config.setParam("AbsenceDataTable",absenceTable);
|
||||
config.setParam("PresenceDataTable",presenceTable);
|
||||
|
||||
config.setParam("SpeciesName",speciesCode);
|
||||
config.setParam("UserName",userName);
|
||||
config.setParam("LayersNeurons",layersNeurons);
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package org.gcube.dataanalysis.ecoengine.utils;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
|
||||
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad;
|
||||
import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources;
|
||||
|
||||
public class ResourceFactory {
|
||||
|
||||
public static String getResources(float... values) {
|
||||
Resources res = new Resources();
|
||||
for (int i = 0; i < values.length; i++)
|
||||
res.addResource("Thread_" + (i + 1), values[i]);
|
||||
|
||||
return HttpRequest.toJSon(res.list).replace("resId", "resID");
|
||||
}
|
||||
|
||||
public static String getLoad(int processedRecords){
|
||||
long tk = System.currentTimeMillis();
|
||||
double activity = processedRecords;
|
||||
ResourceLoad rs = new ResourceLoad(tk, activity);
|
||||
return rs.toString();
|
||||
}
|
||||
|
||||
long lastTime;
|
||||
int lastProcessedRecordsNumber;
|
||||
public ResourceFactory(){
|
||||
lastTime = System.currentTimeMillis();
|
||||
lastProcessedRecordsNumber = 0;
|
||||
}
|
||||
|
||||
|
||||
public String getResourceLoad(int processedRecords){
|
||||
long tk = System.currentTimeMillis();
|
||||
long timediff = tk - lastTime;
|
||||
if ( timediff == 0)
|
||||
timediff = 1;
|
||||
|
||||
double activity = Double.valueOf(processedRecords - lastProcessedRecordsNumber) * 1000.00 / Double.valueOf(timediff);
|
||||
lastTime = tk;
|
||||
lastProcessedRecordsNumber = processedRecords;
|
||||
ResourceLoad rs = new ResourceLoad(tk, activity);
|
||||
return rs.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
package org.gcube.dataanalysis.ecoengine.utils;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class TrainingSetsGenerator {
|
||||
|
||||
static String getProbabilititesQuery = "select count(*) as distribprob from %1$s as a join %2$s as b on a.%3$s=b.%4$s and b.%5$s %6$s %7$s";
|
||||
static String getNumberOfElementsQuery = "select count(*) from %1$s";
|
||||
|
||||
static String getRandomElements = "select * into %2$s from %1$s order by RANDOM() limit %3$s";
|
||||
static String getOtherElements = "select * into %4$s from %1$s where %3$s not in (select %3$s from %2$s)";
|
||||
static String dropTable = "drop table %1$s";
|
||||
|
||||
float threshold = 0.1f;
|
||||
String configPath = "./cfg/";
|
||||
|
||||
public HashMap<String, VarCouple> getInputParameters() {
|
||||
|
||||
HashMap<String, VarCouple> parameters = new HashMap<String, VarCouple>();
|
||||
|
||||
parameters.put("casesTable", new VarCouple(VARTYPE.STRING, ""));
|
||||
parameters.put("columnKeyName", new VarCouple(VARTYPE.STRING, ""));
|
||||
|
||||
parameters.put("DatabaseUserName", new VarCouple(VARTYPE.DATABASEUSERNAME, ""));
|
||||
parameters.put("DatabasePassword", new VarCouple(VARTYPE.DATABASEPASSWORD, ""));
|
||||
parameters.put("DatabaseURL", new VarCouple(VARTYPE.DATABASEURL, ""));
|
||||
parameters.put("DatabaseDriver", new VarCouple(VARTYPE.DATABASEDRIVER, ""));
|
||||
|
||||
return parameters;
|
||||
}
|
||||
|
||||
private int calculateNumberOfPoints(String table) {
|
||||
|
||||
String numberOfPositiveCasesQuery = String.format(getNumberOfElementsQuery, table);
|
||||
List<Object> totalPoints = DatabaseFactory.executeSQLQuery(numberOfPositiveCasesQuery, connection);
|
||||
int points = Integer.parseInt("" + totalPoints.get(0));
|
||||
return points;
|
||||
}
|
||||
|
||||
private void deleteTable(String testTable, String trainingTable) {
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(String.format(dropTable, testTable), connection);
|
||||
} catch (Exception e) {
|
||||
}
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(String.format(dropTable, trainingTable), connection);
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
|
||||
private void generatePointsTable(String table, String key, int numberOfElements) {
|
||||
String testTable = table + "_test" + numberOfElements;
|
||||
String trainingTable = table + "_training" + numberOfElements;
|
||||
deleteTable(testTable, trainingTable);
|
||||
|
||||
String testSetQuery = String.format(getRandomElements, table, testTable, numberOfElements);
|
||||
AnalysisLogger.getLogger().trace("TestSet Creation Query: "+testSetQuery);
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(testSetQuery, connection);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
String trainingSetQuery = String.format(getOtherElements, table, testTable, key, trainingTable);
|
||||
AnalysisLogger.getLogger().trace("TrainingSet Creation Query: "+trainingSetQuery);
|
||||
try {
|
||||
DatabaseFactory.executeSQLUpdate(trainingSetQuery, connection);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
AnalysisLogger.getLogger().trace("DONE!");
|
||||
}
|
||||
|
||||
public void generate(AlgorithmConfiguration config) throws Exception {
|
||||
init(config);
|
||||
int numberOfElements = calculateNumberOfPoints(config.getParam("casesTable"));
|
||||
int elementsToTake = (int) (0.4 * (float) numberOfElements);
|
||||
generatePointsTable(config.getParam("casesTable"), config.getParam("columnKeyName"), elementsToTake);
|
||||
|
||||
}
|
||||
SessionFactory connection;
|
||||
|
||||
public void init(AlgorithmConfiguration config) throws Exception {
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
// init db connection
|
||||
connection = AlgorithmConfiguration.getConnectionFromConfig(config);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue