This commit is contained in:
Gianpaolo Coro 2014-03-28 17:53:45 +00:00
parent 678a68060d
commit 6c670c89ee
8 changed files with 574 additions and 23 deletions

View File

@ -17,6 +17,7 @@ public class DateGuesser {
System.out.println("pattern "+getPattern("2009-05-12") );
System.out.println("pattern "+getPattern("prova") );
System.out.println("pattern "+getPattern("9/26/2010 1:49:00"));
System.out.println("pattern "+getPattern("0.99"));
c = convertDate("2009-05-12");
System.out.println("giorno " + c.get(Calendar.DAY_OF_MONTH) + " mese " + (c.get(Calendar.MONTH) + 1) + " anno " + c.get(Calendar.YEAR));
// Properties prop = new Properties(System.getProperties());
@ -32,7 +33,7 @@ public class DateGuesser {
}
// private static final String[] formats = { "MM\\dd\\yyyy", "MM\\dd\\yy", "MM/dd/yy", "MM/dd/yyyy", "dd/MM/yy", "dd/MM/yyyy", "dd/MM/yyyy HH:mm:ss", "dd/MM/yy HH:mm:ss", "dd/MM/yyyy HH:mm:ss","MM/yy","MM/yyyy", "yyyy.MM.dd G 'at' HH:mm:ss z", "EEE, MMM d, ''yy", "h:mm a", "hh 'o''clock' a, zzzz", "K:mm a, z", "MM-dd-yy","MM-dd-yyyy", "dd-MMM-yy", "yyyy.MM.dd.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd HH:mm","yyyy-MM-dd","yyyy-MM-dd HH:mm:ss", "h:mm a", "yyyy"};
private static final String[] formats = { "MM\\dd\\yyyy", "MM\\dd\\yy", "MM/dd/yy", "MM/dd/yyyy", "MM/yy","MM/yyyy", "yyyy.MM.dd G 'at' HH:mm:ss z", "MM/dd/yyyy HH:mm:ss","dd/MM/yyyy HH:mm:ss","EEE, MMM d, ''yy", "h:mm a", "hh 'o''clock' a, zzzz", "K:mm a, z", "MM-dd-yy","MM-dd-yyyy", "dd-MMM-yy", "yyyy.MM.dd.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd HH:mm","yyyy-MM-dd","yyyy-MM-dd HH:mm:ss","MM/dd/yy KK:mm a","MM/dd/yy KK:mm:ss a","h:mm a", "yyyy"};
private static final String[] formats = { "MM\\dd\\yyyy", "MM\\dd\\yy", "MM/dd/yy", "MM/dd/yyyy", "MM/yy","MM/yyyy", "yyyy.MM.dd G 'at' HH:mm:ss z", "MM/dd/yyyy HH:mm:ss","dd/MM/yyyy HH:mm:ss","EEE, MMM d, ''yy", "h:mm a", "hh 'o''clock' a, zzzz", "K:mm a, z", "MM-dd-yy","MM-dd-yyyy", "dd-MMM-yy", "yyyy.MM.dd.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd HH:mm","yyyy-MM-dd","yyyy-MM-dd HH:mm:ss","MM/dd/yy KK:mm a","MM/dd/yy KK:mm:ss a","h:mm a", "yyyy","s"};
private static final String[] formatiITA = { "dd\\MM\\yyyy", "dd\\MM\\yy", "dd/MM/yy", "dd/MM/yyyy", "dd/MM/yy", "dd/MM/yyyy","dd/MM/yyyy HH:mm:ss", "dd/MM/yy HH:mm:ss", "dd/MM/yyyy HH:mm:ss", "MM/yy","MM/yyyy","dd.MM.yyyy G 'alle' HH:mm:ss z", "EEE, MMM d, ''yy", "h:mm a", "hh a, zzzz", "K:mm a, z", "dd-MMM-yy", "dd.MM.yyyy.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "dd-MMM-yyyy HH:mm", "h:mm a", "yyyy"};

View File

@ -115,7 +115,9 @@ public class PeriodicityDetector {
int windowAnalysisSamples = (int) Math.pow(2, 14);// (int)
windowAnalysisSamples = wLength;
int windowShiftSamples = (int) Math.round((float) windowAnalysisSamples / 2f);
float windowShiftTime = (float)SignalConverter.sample2Time(windowShiftSamples, samplingRate);
float error = ((float) samplingRate / (float) windowAnalysisSamples);
AnalysisLogger.getLogger().debug("Error in the Measure will be: " + error + " Hz");
@ -130,8 +132,8 @@ public class PeriodicityDetector {
minFrequency=minPossibleFreq;
maxFrequency=maxPossibleFreq;
// display the signal
if (display)
SignalProcessing.displaySignalWithGenericTime(signal, 0, 1, "signal");
// if (display)
// SignalProcessing.displaySignalWithGenericTime(signal, 0, 1, "signal");
this.currentSamplingRate = samplingRate;
this.currentWindowShiftSamples = windowShiftSamples;
@ -160,8 +162,8 @@ public class PeriodicityDetector {
if (maxfrequencies == null)
return 0;
this.startPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.startStableTractIdx, samplingRate);
this.endPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.endStableTractIdx, samplingRate);
this.startPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.startStableTractIdx, windowShiftTime);
this.endPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.endStableTractIdx, windowShiftTime);
this.startPeriodSampleIndex=SignalConverter.time2Sample(startPeriodTime, samplingRate);
this.endPeriodSampleIndex=Math.min(SignalConverter.time2Sample(endPeriodTime, samplingRate),signal.length-1);

View File

@ -7,18 +7,16 @@ import java.util.Arrays;
import javax.swing.JPanel;
import org.gcube.contentmanagement.graphtools.data.BigSamplesTable;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.utils.Operations;
import marytts.signalproc.display.SpectrogramCustom;
import marytts.signalproc.window.Window;
import org.gcube.contentmanagement.graphtools.data.BigSamplesTable;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import com.rapidminer.example.Attribute;
import com.rapidminer.example.Example;
import com.rapidminer.example.ExampleSet;
import com.rapidminer.example.table.MemoryExampleTable;
import com.rapidminer.tools.math.MathFunctions;
/**
* includes tools for basic signal transformations:

View File

@ -172,9 +172,17 @@ public class SignalProcessing {
public static void displaySignalWithTime(double[] signal, Date[] dates, String name, String format) {
org.jfree.data.time.TimeSeries series = new org.jfree.data.time.TimeSeries(name);
for (int i = 0; i < signal.length; i++)
series.add(new FixedMillisecond(dates[i]), signal[i]);
if (signal.length>20000) {
AnalysisLogger.getLogger().debug("Too many points to display: "+signal.length);
return;
}
for (int i = 0; i < signal.length; i++){
try{
series.add(new FixedMillisecond(dates[i]), signal[i]);
}catch(Exception e){
AnalysisLogger.getLogger().debug("Skipping value yet present: "+dates[i]);
}
}
TimeSeriesCollection dataset = new TimeSeriesCollection();
dataset.addSeries(series);
@ -232,18 +240,15 @@ public class SignalProcessing {
return sumSpectro;
}
public static Date[] fillTimeLine(double[] timeseconds, double samplingRate, AlgorithmConfiguration config) throws Exception {
double[] values = new double[timeseconds.length];
for (int i = 0; i < timeseconds.length; i++)
values[i] = timeseconds[i] * 1000d;
double[] milliseconds = fillTimeSeries(values, timeseconds, samplingRate, config);
public static Date[] fillTimeLine(double[] timemilliseconds, double samplingRate, AlgorithmConfiguration config) throws Exception {
double[] milliseconds = fillTimeSeries(timemilliseconds, timemilliseconds, samplingRate, config);
Date[] dates = new Date[milliseconds.length];
for (int i = 0; i < milliseconds.length; i++)
dates[i] = new Date((long) milliseconds[i]);
return dates;
}
public static double[] fillSignal(double[] signal) throws Exception{
ExampleSet es = SignalConverter.signal2ExampleSet(signal);
SeriesMissingValueReplenishment sampler = (SeriesMissingValueReplenishment) OperatorService.createOperator("SeriesMissingValueReplenishment");
@ -258,15 +263,20 @@ public class SignalProcessing {
}
public static double[] fillTimeSeries(double[] values, double[] timeseconds, double samplingRate, AlgorithmConfiguration config) throws Exception {
config.initRapidMiner();
double t0 = timeseconds[0];
double t1 = timeseconds[timeseconds.length - 1];
int signalength = (int) ((t1 - t0) * samplingRate) + 1;
int signalength = Math.abs((int) ((t1 - t0) * samplingRate) + 1);
AnalysisLogger.getLogger().debug("SignalProcessing->Old Time Series had: "+values.length+" samples. New Time Series will have: "+signalength+" samples");
if (values.length==signalength)
return values;
config.initRapidMiner();
double signal[] = new double[signalength];
Arrays.fill(signal, Double.NaN);
for (int i = 0; i < values.length; i++) {
if (values[i] != Double.NaN) {
int index = (int) ((timeseconds[i] - t0) * samplingRate);
int index = Math.abs((int) ((timeseconds[i] - t0) * samplingRate));
signal[index] = values[i];
}
}
@ -291,6 +301,7 @@ public class SignalProcessing {
return rebuiltSignal;
}
public static double[][] multiSignalAnalysis(List<double[]> signals, int samplingRate, int windowshift, int frameslength, boolean display) throws Exception {
List<double[][]> spectra = new ArrayList<double[][]>();

View File

@ -0,0 +1,237 @@
package org.gcube.dataanalysis.ecoengine.signals;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import org.gcube.contentmanagement.graphtools.utils.DateGuesser;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.utils.Operations;
import org.gcube.dataanalysis.ecoengine.utils.Tuple;
public class TimeSeries {
private double[] values;
private Date[] times;
private Date[] unsortedtimes;
private String[] timeLabels;
private long minimumtimegap = -1;
AlgorithmConfiguration config;
public TimeSeries(int timeLength, AlgorithmConfiguration config) {
values = new double[timeLength];
times = new Date[timeLength];
unsortedtimes = new Date[timeLength];
timeLabels = new String[timeLength];
this.config = config;
}
public TimeSeries(double[] values, Date[] time, String[] timeLabels, AlgorithmConfiguration config) {
this.values = values;
this.times = time;
this.unsortedtimes = Arrays.copyOf(time, time.length);
this.timeLabels = timeLabels;
this.config = config;
}
public void setValues(double[] values) {
this.values = values;
}
public void setTime(Date[] time) {
this.times = time;
}
public void setTimeLabels(String[] timeLabels) {
this.timeLabels = timeLabels;
}
public double[] getValues() {
return values;
}
public String[] getLabels() {
return timeLabels;
}
public Date[] getTime() {
return times;
}
public double[] getMillisecondsTimeline() {
double[] secondstimes = new double[times.length];
for (int i = 0; i < times.length; i++) {
long t = times[i].getTime();
secondstimes[i] = (double) t;
}
return secondstimes;
}
public long getMimimumTimeGapInMillisecs() {
if (minimumtimegap > -1)
return minimumtimegap;
long mintime = Long.MAX_VALUE;
for (int i = 1; i < times.length; i++) {
long t0 = times[i - 1].getTime();
long t1 = times[i].getTime();
long timediff = Math.abs(t1 - t0);
if (timediff < mintime && timediff > 0)
mintime = timediff;
}
minimumtimegap = mintime;
return mintime;
}
public void addElement(double value, Date date, String label, int index) {
values[index] = value;
times[index] = date;
unsortedtimes[index] = date;
timeLabels[index] = label;
}
public void sort() {
Arrays.sort(times);
double[] tempvalues = new double[values.length];
String[] temptimeLabels = new String[timeLabels.length];
List<Date> unsortedTimesList = Arrays.asList(unsortedtimes);
int i = 0;
for (Date time : times) {
int index = unsortedTimesList.indexOf(time);
tempvalues[i] = values[index];
temptimeLabels[i] = timeLabels[index];
i++;
}
values = null;
timeLabels = null;
values = tempvalues;
timeLabels = temptimeLabels;
unsortedtimes = Arrays.copyOf(times, times.length);
}
public double getValue(int index) {
return values[index];
}
public Date getDate(int index) {
return times[index];
}
public String getTimeLabel(int index) {
return timeLabels[index];
}
// each element in the list is Time,Quantity
public static TimeSeries buildFromSignal(List<Tuple<String>> lines, AlgorithmConfiguration config) {
TimeSeries ts = new TimeSeries(lines.size(), config);
int counter = 0;
HashMap<String, String> timescache = new HashMap<String, String>();
String timepattern = null;
boolean isSeconds = false;
for (Tuple<String> line : lines) {
String timel = line.getElements().get(0);
timel = timel.replace("time:", "");
Double quantity = Double.parseDouble(line.getElements().get(1));
Date time = null;
/*
* try { SimpleDateFormat sdf = new SimpleDateFormat("E MMM dd HH:mm:ss zzz yyyy", Locale.ENGLISH); time = (Date) sdf.parse(timel); } catch (Exception e) { } try { SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.ROOT); time = (Date) sdf.parse(timel); } catch (Exception e) { }
*
* if (time == null)
*/
// time = DateGuesser.convertDate(timel).getTime();
try {
if (counter == 0) {
timepattern = DateGuesser.getPattern(timel);
AnalysisLogger.getLogger().debug("Time pattern: " + timepattern);
//distinguish between years and seconds
double seconds =-1;
try{
seconds = Double.parseDouble(timel);
int secondsint = (int) seconds;
if (((""+secondsint).length()==4) && (secondsint==seconds))
isSeconds = false;
else{
AnalysisLogger.getLogger().debug("Getting seconds instead of pattern!: " + timel);
isSeconds = true;
timepattern="s";
}
}catch(Exception e1){}
}
SimpleDateFormat sdf = new SimpleDateFormat(timepattern, Locale.ENGLISH);
time = (Date) sdf.parse(timel);
} catch (Exception e) {
time = DateGuesser.convertDate(timel).getTime();
AnalysisLogger.getLogger().debug("ERROR in parsing time :"+timel+" adjusting...");
// e.printStackTrace();
}
if (counter == 0) {
AnalysisLogger.getLogger().debug("Date detection: input " + timel + " output " + time);
}
ts.addElement(quantity, time, timel, counter);
counter++;
}
ts.sort();
return ts;
}
public void convertToUniformSignal(double samplingrate) throws Exception {
if (samplingrate <= 0) {
if (minimumtimegap < 0)
getMimimumTimeGapInMillisecs();
if (minimumtimegap > 0)
samplingrate = 1d / (double) minimumtimegap;
}
AnalysisLogger.getLogger().debug("TimeSeries->Samplig rate: " + samplingrate + " minimum gap in time: " + minimumtimegap);
if (samplingrate == 0)
return;
double[] timeline = getMillisecondsTimeline();
AnalysisLogger.getLogger().debug("TimeSeries->filling gaps");
double[] newvalues = SignalProcessing.fillTimeSeries(values, timeline, samplingrate, config);
if (newvalues.length != values.length) {
AnalysisLogger.getLogger().debug("TimeSeries->filling also time values");
Date[] newtimeline = SignalProcessing.fillTimeLine(timeline, samplingrate, config);
values = null;
times = null;
unsortedtimes = null;
values = newvalues;
times = newtimeline;
unsortedtimes = newtimeline;
timeLabels = new String[times.length];
}
AnalysisLogger.getLogger().debug("TimeSeries->Returning values");
timeLabels = new String[times.length];
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.ROOT);
for (int i = 0; i < times.length; i++) {
timeLabels[i] = sdf.format(times[i]);
}
}
public void normalize() throws Exception {
double max = Operations.getMax(values);
for (int i = 0; i < values.length; i++) {
values[i] = values[i] / max;
}
}
}

View File

@ -0,0 +1,150 @@
package org.gcube.dataanalysis.ecoengine.test.signalprocessing;
import java.util.List;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
public class TestSimpleSignal {
// static AlgorithmConfiguration[] configs = {periodicSignalConfig()};//russianSignalConfig(),simpleSignalConfig()};
static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig(), occurrencePointsSignalConfig(),hugeSignalConfig()};
// static AlgorithmConfiguration[] configs = {russianSignalConfig()};
// static AlgorithmConfiguration[] configs = {hugeSignalConfig()};
// static AlgorithmConfiguration[] configs = {periodicSignalConfig()};
public static void main(String[] args) throws Exception {
for (int i = 0; i < configs.length; i++) {
System.out.println("*****************TEST "+i+" *****************");
List<ComputationalAgent> trans = null;
trans = TransducerersFactory.getTransducerers(configs[i]);
trans.get(0).init();
Regressor.process(trans.get(0));
StatisticalType st = trans.get(0).getOutput();
AnalysisLogger.getLogger().debug("ST:" + st);
trans = null;
System.out.println("*****************END TEST*****************");
}
}
public static AlgorithmConfiguration simpleSignalConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("TIME_SERIES_PROCESSING");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "timeseries_id4dd368bf_63fb_4d19_8e31_20ced63a477d");
config.setParam("ValueColum", "quantity");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setGcubeScope("/gcube");
config.setConfigPath("./cfg");
return config;
}
public static AlgorithmConfiguration russianSignalConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("TIME_SERIES_PROCESSING");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "generic_ideb9efbe0_61ad_4eea_b0ee_95e64ce11b28");
config.setParam("ValueColum", "quantity");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setGcubeScope("/gcube");
config.setConfigPath("./cfg");
return config;
}
public static AlgorithmConfiguration occurrencePointsSignalConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("TIME_SERIES_PROCESSING");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "generic_id037d302d_2ba0_4e43_b6e4_1a797bb91728");
config.setParam("ValueColum", "speed");
config.setParam("TimeColum", "datetime");
config.setParam("AggregationFunction", "AVG");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setGcubeScope("/gcube");
config.setConfigPath("./cfg");
return config;
}
public static AlgorithmConfiguration periodicSignalConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("TIME_SERIES_PROCESSING");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "signalcsv");
config.setParam("ValueColum", "signal");
config.setParam("FrequencyResolution", "0.1");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setGcubeScope("/gcube");
config.setConfigPath("./cfg");
return config;
}
public static AlgorithmConfiguration hugeSignalConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("TIME_SERIES_PROCESSING");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "generic_id634a660c_4d1a_410c_aa45_eb6e4c5afdf9");
config.setParam("ValueColum", "quantity");
config.setParam("TimeColum", "years");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setGcubeScope("/gcube");
config.setConfigPath("./cfg");
return config;
}
}

View File

@ -0,0 +1,145 @@
package org.gcube.dataanalysis.ecoengine.transducers;
import java.util.ArrayList;
import java.util.List;
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
import org.gcube.dataanalysis.ecoengine.signals.PeriodicityDetector;
import org.gcube.dataanalysis.ecoengine.signals.SignalProcessing;
import org.gcube.dataanalysis.ecoengine.signals.TimeSeries;
import org.gcube.dataanalysis.ecoengine.utils.AggregationFunctions;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
import org.gcube.dataanalysis.ecoengine.utils.Tuple;
import org.hibernate.SessionFactory;
public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm {
private static String timeSeriesTable = "TimeSeriesTable";
private static String valuesColumn= "ValueColum";
private static String timeColumn= "TimeColum";
private static String frequencyResolution= "FrequencyResolution";
private static String aggregationFunction= "AggregationFunction";
@Override
public void init() throws Exception {
}
@Override
public String getDescription() {
return "An algorithms applying signal processing to a non uniform time series. It uniformly samples the series, then extracts hidden periodicities and signal properties.";
}
@Override
protected void process() throws Exception {
SessionFactory dbconnection = null;
try{
dbconnection = DatabaseUtils.initDBSession(config);
String tablename = config.getParam(timeSeriesTable);
String valuescolum = config.getParam(valuesColumn);
// String timecolumn = "datetime";//"time";
String timecolumn = config.getParam(timeColumn);
String aggregationFunc= config.getParam(aggregationFunction);
String frequencyRes= config.getParam(frequencyResolution);
float frequencyResDouble = 1;
if (timecolumn==null)
timecolumn = "time";
if (aggregationFunc==null)
aggregationFunc="SUM";
if (frequencyRes!=null)
{ try{
frequencyResDouble=Float.parseFloat(frequencyRes);
}catch(Exception e){
}
}
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Table Name: "+tablename);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Time Column: "+timecolumn);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Values Column: "+valuescolum);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Aggregation: "+aggregationFunc);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Frequency Resolution: "+frequencyRes);
// String query = "select * from (select "+valuescolum+","+timecolumn+" from "+tablename+") as a";
String query = "select * from (select "+aggregationFunc+"("+valuescolum+"),"+timecolumn+" from "+tablename+" group by "+timecolumn+") as a";
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Query to execute: "+query);
List<Object> results = DatabaseFactory.executeSQLQuery(query, dbconnection);
if (results==null || results.size()==0)
throw new Exception("Error in retrieving values from the table: no time series found");
//build signal
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Building signal");
List<Tuple<String>> signal = new ArrayList<Tuple<String>>();
int sizesignal = 0;
for (Object row:results){
Object[] srow = (Object[]) row;
String value =""+srow[0];
String time=""+srow[1];
signal.add(new Tuple<String>(time,value));
sizesignal++;
}
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Signal built with success. Size: "+sizesignal);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Building Time Series");
TimeSeries ts = TimeSeries.buildFromSignal(signal, config);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniformly sampling the signal");
SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Time Series", "HH:mm:ss MM-dd-yy");
ts.convertToUniformSignal(0);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniform sampling finished");
//spectrum and signal processing
SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Unif Samp Time Series", "HH:mm:ss MM-dd-yy");
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Detecting periodicity");
PeriodicityDetector pd = new PeriodicityDetector();
double F = pd.detectFrequency(ts.getValues(),1,0.01f,0.5f,frequencyResDouble,true);
AnalysisLogger.getLogger().debug("Detected Frequency: "+F+" indecision ["+pd.lowermeanF+" , "+pd.uppermeanF+"]");
AnalysisLogger.getLogger().debug("Detected Period: "+pd.meanPeriod+" indecision ["+pd.lowermeanPeriod+" , "+pd.uppermeanPeriod+"]");
AnalysisLogger.getLogger().debug("Detected Periodicity Strength: "+pd.periodicityStrength+" ("+pd.getPeriodicityStregthInterpretation()+")");
AnalysisLogger.getLogger().debug("Periodicity inside this samples range: ["+pd.startPeriodTime+";"+pd.endPeriodTime+"]");
AnalysisLogger.getLogger().debug("Maximum Frequency in the Spectrogram "+MathFunctions.roundDecimal(pd.maxFrequency,2));
AnalysisLogger.getLogger().debug("Minimum Frequency in the Spectrogram "+MathFunctions.roundDecimal(pd.minFrequency,2));
// Image spectrogramImage = SignalProcessing.renderSignalSpectrogram2(pd.currentspectrum);
}catch(Exception e){
e.printStackTrace();
throw e;
}finally{
if (dbconnection!=null)
dbconnection.close();
}
}
@Override
protected void setInputParameters() {
//the time series table
List<TableTemplates> templates = new ArrayList<TableTemplates>();
templates.add(TableTemplates.TIMESERIES);
InputTable p = new InputTable(templates, timeSeriesTable, "The table containing the time series", "timeseries");
inputs.add(p);
ColumnType p1 = new ColumnType(timeSeriesTable, valuesColumn, "The column containing the values of the time series", "values", false);
inputs.add(p1);
addDoubleInput(frequencyResolution, "The precision in detecting the period. The lower this number the lower the number of samples used at each step in the Spectrogram. Reducing this, the spectrogram will be finer and sharper, but you should tune it. Too few samples will make the Spectrogram noisy.", "1");
addEnumerateInput(AggregationFunctions.values(), aggregationFunction, "Function to apply to samples with the same time instant", AggregationFunctions.SUM.name());
}
@Override
public void shutdown() {
}
}

View File

@ -0,0 +1,7 @@
package org.gcube.dataanalysis.ecoengine.utils;
public enum AggregationFunctions {
SUM,
AVG
}