git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@93806 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
678a68060d
commit
6c670c89ee
|
|
@ -17,6 +17,7 @@ public class DateGuesser {
|
|||
System.out.println("pattern "+getPattern("2009-05-12") );
|
||||
System.out.println("pattern "+getPattern("prova") );
|
||||
System.out.println("pattern "+getPattern("9/26/2010 1:49:00"));
|
||||
System.out.println("pattern "+getPattern("0.99"));
|
||||
c = convertDate("2009-05-12");
|
||||
System.out.println("giorno " + c.get(Calendar.DAY_OF_MONTH) + " mese " + (c.get(Calendar.MONTH) + 1) + " anno " + c.get(Calendar.YEAR));
|
||||
// Properties prop = new Properties(System.getProperties());
|
||||
|
|
@ -32,7 +33,7 @@ public class DateGuesser {
|
|||
}
|
||||
|
||||
// private static final String[] formats = { "MM\\dd\\yyyy", "MM\\dd\\yy", "MM/dd/yy", "MM/dd/yyyy", "dd/MM/yy", "dd/MM/yyyy", "dd/MM/yyyy HH:mm:ss", "dd/MM/yy HH:mm:ss", "dd/MM/yyyy HH:mm:ss","MM/yy","MM/yyyy", "yyyy.MM.dd G 'at' HH:mm:ss z", "EEE, MMM d, ''yy", "h:mm a", "hh 'o''clock' a, zzzz", "K:mm a, z", "MM-dd-yy","MM-dd-yyyy", "dd-MMM-yy", "yyyy.MM.dd.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd HH:mm","yyyy-MM-dd","yyyy-MM-dd HH:mm:ss", "h:mm a", "yyyy"};
|
||||
private static final String[] formats = { "MM\\dd\\yyyy", "MM\\dd\\yy", "MM/dd/yy", "MM/dd/yyyy", "MM/yy","MM/yyyy", "yyyy.MM.dd G 'at' HH:mm:ss z", "MM/dd/yyyy HH:mm:ss","dd/MM/yyyy HH:mm:ss","EEE, MMM d, ''yy", "h:mm a", "hh 'o''clock' a, zzzz", "K:mm a, z", "MM-dd-yy","MM-dd-yyyy", "dd-MMM-yy", "yyyy.MM.dd.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd HH:mm","yyyy-MM-dd","yyyy-MM-dd HH:mm:ss","MM/dd/yy KK:mm a","MM/dd/yy KK:mm:ss a","h:mm a", "yyyy"};
|
||||
private static final String[] formats = { "MM\\dd\\yyyy", "MM\\dd\\yy", "MM/dd/yy", "MM/dd/yyyy", "MM/yy","MM/yyyy", "yyyy.MM.dd G 'at' HH:mm:ss z", "MM/dd/yyyy HH:mm:ss","dd/MM/yyyy HH:mm:ss","EEE, MMM d, ''yy", "h:mm a", "hh 'o''clock' a, zzzz", "K:mm a, z", "MM-dd-yy","MM-dd-yyyy", "dd-MMM-yy", "yyyy.MM.dd.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyy-MM-dd HH:mm","yyyy-MM-dd","yyyy-MM-dd HH:mm:ss","MM/dd/yy KK:mm a","MM/dd/yy KK:mm:ss a","h:mm a", "yyyy","s"};
|
||||
private static final String[] formatiITA = { "dd\\MM\\yyyy", "dd\\MM\\yy", "dd/MM/yy", "dd/MM/yyyy", "dd/MM/yy", "dd/MM/yyyy","dd/MM/yyyy HH:mm:ss", "dd/MM/yy HH:mm:ss", "dd/MM/yyyy HH:mm:ss", "MM/yy","MM/yyyy","dd.MM.yyyy G 'alle' HH:mm:ss z", "EEE, MMM d, ''yy", "h:mm a", "hh a, zzzz", "K:mm a, z", "dd-MMM-yy", "dd.MM.yyyy.HH.mm.ss", "E, dd MMM yyyy HH:mm:ss Z", "yyyyy.MMMMM.dd GGG hh:mm aaa", "EEE, d MMM yyyy HH:mm:ss Z", "yyMMddHHmmssZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "dd-MMM-yyyy HH:mm", "h:mm a", "yyyy"};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -115,7 +115,9 @@ public class PeriodicityDetector {
|
|||
int windowAnalysisSamples = (int) Math.pow(2, 14);// (int)
|
||||
windowAnalysisSamples = wLength;
|
||||
int windowShiftSamples = (int) Math.round((float) windowAnalysisSamples / 2f);
|
||||
|
||||
float windowShiftTime = (float)SignalConverter.sample2Time(windowShiftSamples, samplingRate);
|
||||
|
||||
|
||||
float error = ((float) samplingRate / (float) windowAnalysisSamples);
|
||||
|
||||
AnalysisLogger.getLogger().debug("Error in the Measure will be: " + error + " Hz");
|
||||
|
|
@ -130,8 +132,8 @@ public class PeriodicityDetector {
|
|||
minFrequency=minPossibleFreq;
|
||||
maxFrequency=maxPossibleFreq;
|
||||
// display the signal
|
||||
if (display)
|
||||
SignalProcessing.displaySignalWithGenericTime(signal, 0, 1, "signal");
|
||||
// if (display)
|
||||
// SignalProcessing.displaySignalWithGenericTime(signal, 0, 1, "signal");
|
||||
|
||||
this.currentSamplingRate = samplingRate;
|
||||
this.currentWindowShiftSamples = windowShiftSamples;
|
||||
|
|
@ -160,8 +162,8 @@ public class PeriodicityDetector {
|
|||
if (maxfrequencies == null)
|
||||
return 0;
|
||||
|
||||
this.startPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.startStableTractIdx, samplingRate);
|
||||
this.endPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.endStableTractIdx, samplingRate);
|
||||
this.startPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.startStableTractIdx, windowShiftTime);
|
||||
this.endPeriodTime = SignalConverter.spectrogramTimeFromIndex(signalconverter.endStableTractIdx, windowShiftTime);
|
||||
this.startPeriodSampleIndex=SignalConverter.time2Sample(startPeriodTime, samplingRate);
|
||||
this.endPeriodSampleIndex=Math.min(SignalConverter.time2Sample(endPeriodTime, samplingRate),signal.length-1);
|
||||
|
||||
|
|
|
|||
|
|
@ -7,18 +7,16 @@ import java.util.Arrays;
|
|||
|
||||
import javax.swing.JPanel;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.data.BigSamplesTable;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.Operations;
|
||||
|
||||
import marytts.signalproc.display.SpectrogramCustom;
|
||||
import marytts.signalproc.window.Window;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.data.BigSamplesTable;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
|
||||
import com.rapidminer.example.Attribute;
|
||||
import com.rapidminer.example.Example;
|
||||
import com.rapidminer.example.ExampleSet;
|
||||
import com.rapidminer.example.table.MemoryExampleTable;
|
||||
import com.rapidminer.tools.math.MathFunctions;
|
||||
|
||||
/**
|
||||
* includes tools for basic signal transformations:
|
||||
|
|
|
|||
|
|
@ -172,9 +172,17 @@ public class SignalProcessing {
|
|||
|
||||
public static void displaySignalWithTime(double[] signal, Date[] dates, String name, String format) {
|
||||
org.jfree.data.time.TimeSeries series = new org.jfree.data.time.TimeSeries(name);
|
||||
for (int i = 0; i < signal.length; i++)
|
||||
series.add(new FixedMillisecond(dates[i]), signal[i]);
|
||||
|
||||
if (signal.length>20000) {
|
||||
AnalysisLogger.getLogger().debug("Too many points to display: "+signal.length);
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < signal.length; i++){
|
||||
try{
|
||||
series.add(new FixedMillisecond(dates[i]), signal[i]);
|
||||
}catch(Exception e){
|
||||
AnalysisLogger.getLogger().debug("Skipping value yet present: "+dates[i]);
|
||||
}
|
||||
}
|
||||
TimeSeriesCollection dataset = new TimeSeriesCollection();
|
||||
dataset.addSeries(series);
|
||||
|
||||
|
|
@ -232,18 +240,15 @@ public class SignalProcessing {
|
|||
return sumSpectro;
|
||||
}
|
||||
|
||||
public static Date[] fillTimeLine(double[] timeseconds, double samplingRate, AlgorithmConfiguration config) throws Exception {
|
||||
double[] values = new double[timeseconds.length];
|
||||
for (int i = 0; i < timeseconds.length; i++)
|
||||
values[i] = timeseconds[i] * 1000d;
|
||||
double[] milliseconds = fillTimeSeries(values, timeseconds, samplingRate, config);
|
||||
public static Date[] fillTimeLine(double[] timemilliseconds, double samplingRate, AlgorithmConfiguration config) throws Exception {
|
||||
double[] milliseconds = fillTimeSeries(timemilliseconds, timemilliseconds, samplingRate, config);
|
||||
Date[] dates = new Date[milliseconds.length];
|
||||
for (int i = 0; i < milliseconds.length; i++)
|
||||
dates[i] = new Date((long) milliseconds[i]);
|
||||
|
||||
return dates;
|
||||
}
|
||||
|
||||
|
||||
public static double[] fillSignal(double[] signal) throws Exception{
|
||||
ExampleSet es = SignalConverter.signal2ExampleSet(signal);
|
||||
SeriesMissingValueReplenishment sampler = (SeriesMissingValueReplenishment) OperatorService.createOperator("SeriesMissingValueReplenishment");
|
||||
|
|
@ -258,15 +263,20 @@ public class SignalProcessing {
|
|||
}
|
||||
|
||||
public static double[] fillTimeSeries(double[] values, double[] timeseconds, double samplingRate, AlgorithmConfiguration config) throws Exception {
|
||||
config.initRapidMiner();
|
||||
|
||||
double t0 = timeseconds[0];
|
||||
double t1 = timeseconds[timeseconds.length - 1];
|
||||
int signalength = (int) ((t1 - t0) * samplingRate) + 1;
|
||||
int signalength = Math.abs((int) ((t1 - t0) * samplingRate) + 1);
|
||||
AnalysisLogger.getLogger().debug("SignalProcessing->Old Time Series had: "+values.length+" samples. New Time Series will have: "+signalength+" samples");
|
||||
if (values.length==signalength)
|
||||
return values;
|
||||
|
||||
config.initRapidMiner();
|
||||
double signal[] = new double[signalength];
|
||||
Arrays.fill(signal, Double.NaN);
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
if (values[i] != Double.NaN) {
|
||||
int index = (int) ((timeseconds[i] - t0) * samplingRate);
|
||||
int index = Math.abs((int) ((timeseconds[i] - t0) * samplingRate));
|
||||
signal[index] = values[i];
|
||||
}
|
||||
}
|
||||
|
|
@ -291,6 +301,7 @@ public class SignalProcessing {
|
|||
return rebuiltSignal;
|
||||
}
|
||||
|
||||
|
||||
public static double[][] multiSignalAnalysis(List<double[]> signals, int samplingRate, int windowshift, int frameslength, boolean display) throws Exception {
|
||||
|
||||
List<double[][]> spectra = new ArrayList<double[][]>();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,237 @@
|
|||
package org.gcube.dataanalysis.ecoengine.signals;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.utils.DateGuesser;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.Operations;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.Tuple;
|
||||
|
||||
public class TimeSeries {
|
||||
|
||||
private double[] values;
|
||||
private Date[] times;
|
||||
private Date[] unsortedtimes;
|
||||
private String[] timeLabels;
|
||||
private long minimumtimegap = -1;
|
||||
AlgorithmConfiguration config;
|
||||
|
||||
public TimeSeries(int timeLength, AlgorithmConfiguration config) {
|
||||
values = new double[timeLength];
|
||||
times = new Date[timeLength];
|
||||
unsortedtimes = new Date[timeLength];
|
||||
timeLabels = new String[timeLength];
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public TimeSeries(double[] values, Date[] time, String[] timeLabels, AlgorithmConfiguration config) {
|
||||
this.values = values;
|
||||
this.times = time;
|
||||
this.unsortedtimes = Arrays.copyOf(time, time.length);
|
||||
this.timeLabels = timeLabels;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public void setValues(double[] values) {
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
public void setTime(Date[] time) {
|
||||
this.times = time;
|
||||
}
|
||||
|
||||
public void setTimeLabels(String[] timeLabels) {
|
||||
this.timeLabels = timeLabels;
|
||||
}
|
||||
|
||||
public double[] getValues() {
|
||||
return values;
|
||||
}
|
||||
|
||||
public String[] getLabels() {
|
||||
return timeLabels;
|
||||
}
|
||||
|
||||
public Date[] getTime() {
|
||||
return times;
|
||||
}
|
||||
|
||||
public double[] getMillisecondsTimeline() {
|
||||
double[] secondstimes = new double[times.length];
|
||||
for (int i = 0; i < times.length; i++) {
|
||||
long t = times[i].getTime();
|
||||
secondstimes[i] = (double) t;
|
||||
}
|
||||
return secondstimes;
|
||||
}
|
||||
|
||||
public long getMimimumTimeGapInMillisecs() {
|
||||
if (minimumtimegap > -1)
|
||||
return minimumtimegap;
|
||||
|
||||
long mintime = Long.MAX_VALUE;
|
||||
for (int i = 1; i < times.length; i++) {
|
||||
long t0 = times[i - 1].getTime();
|
||||
long t1 = times[i].getTime();
|
||||
long timediff = Math.abs(t1 - t0);
|
||||
if (timediff < mintime && timediff > 0)
|
||||
mintime = timediff;
|
||||
}
|
||||
minimumtimegap = mintime;
|
||||
return mintime;
|
||||
}
|
||||
|
||||
public void addElement(double value, Date date, String label, int index) {
|
||||
values[index] = value;
|
||||
times[index] = date;
|
||||
unsortedtimes[index] = date;
|
||||
timeLabels[index] = label;
|
||||
}
|
||||
|
||||
public void sort() {
|
||||
Arrays.sort(times);
|
||||
double[] tempvalues = new double[values.length];
|
||||
String[] temptimeLabels = new String[timeLabels.length];
|
||||
List<Date> unsortedTimesList = Arrays.asList(unsortedtimes);
|
||||
int i = 0;
|
||||
for (Date time : times) {
|
||||
int index = unsortedTimesList.indexOf(time);
|
||||
tempvalues[i] = values[index];
|
||||
temptimeLabels[i] = timeLabels[index];
|
||||
i++;
|
||||
}
|
||||
values = null;
|
||||
timeLabels = null;
|
||||
values = tempvalues;
|
||||
timeLabels = temptimeLabels;
|
||||
unsortedtimes = Arrays.copyOf(times, times.length);
|
||||
}
|
||||
|
||||
public double getValue(int index) {
|
||||
return values[index];
|
||||
}
|
||||
|
||||
public Date getDate(int index) {
|
||||
return times[index];
|
||||
}
|
||||
|
||||
public String getTimeLabel(int index) {
|
||||
return timeLabels[index];
|
||||
}
|
||||
|
||||
// each element in the list is Time,Quantity
|
||||
public static TimeSeries buildFromSignal(List<Tuple<String>> lines, AlgorithmConfiguration config) {
|
||||
TimeSeries ts = new TimeSeries(lines.size(), config);
|
||||
int counter = 0;
|
||||
HashMap<String, String> timescache = new HashMap<String, String>();
|
||||
String timepattern = null;
|
||||
boolean isSeconds = false;
|
||||
for (Tuple<String> line : lines) {
|
||||
|
||||
String timel = line.getElements().get(0);
|
||||
timel = timel.replace("time:", "");
|
||||
Double quantity = Double.parseDouble(line.getElements().get(1));
|
||||
|
||||
Date time = null;
|
||||
/*
|
||||
* try { SimpleDateFormat sdf = new SimpleDateFormat("E MMM dd HH:mm:ss zzz yyyy", Locale.ENGLISH); time = (Date) sdf.parse(timel); } catch (Exception e) { } try { SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.ROOT); time = (Date) sdf.parse(timel); } catch (Exception e) { }
|
||||
*
|
||||
* if (time == null)
|
||||
*/
|
||||
|
||||
|
||||
|
||||
// time = DateGuesser.convertDate(timel).getTime();
|
||||
|
||||
try {
|
||||
if (counter == 0) {
|
||||
timepattern = DateGuesser.getPattern(timel);
|
||||
AnalysisLogger.getLogger().debug("Time pattern: " + timepattern);
|
||||
//distinguish between years and seconds
|
||||
double seconds =-1;
|
||||
try{
|
||||
seconds = Double.parseDouble(timel);
|
||||
int secondsint = (int) seconds;
|
||||
if (((""+secondsint).length()==4) && (secondsint==seconds))
|
||||
isSeconds = false;
|
||||
else{
|
||||
AnalysisLogger.getLogger().debug("Getting seconds instead of pattern!: " + timel);
|
||||
isSeconds = true;
|
||||
timepattern="s";
|
||||
}
|
||||
}catch(Exception e1){}
|
||||
}
|
||||
|
||||
SimpleDateFormat sdf = new SimpleDateFormat(timepattern, Locale.ENGLISH);
|
||||
time = (Date) sdf.parse(timel);
|
||||
|
||||
} catch (Exception e) {
|
||||
time = DateGuesser.convertDate(timel).getTime();
|
||||
AnalysisLogger.getLogger().debug("ERROR in parsing time :"+timel+" adjusting...");
|
||||
// e.printStackTrace();
|
||||
}
|
||||
|
||||
if (counter == 0) {
|
||||
AnalysisLogger.getLogger().debug("Date detection: input " + timel + " output " + time);
|
||||
}
|
||||
|
||||
ts.addElement(quantity, time, timel, counter);
|
||||
counter++;
|
||||
|
||||
}
|
||||
ts.sort();
|
||||
return ts;
|
||||
}
|
||||
|
||||
public void convertToUniformSignal(double samplingrate) throws Exception {
|
||||
if (samplingrate <= 0) {
|
||||
if (minimumtimegap < 0)
|
||||
getMimimumTimeGapInMillisecs();
|
||||
if (minimumtimegap > 0)
|
||||
samplingrate = 1d / (double) minimumtimegap;
|
||||
}
|
||||
|
||||
AnalysisLogger.getLogger().debug("TimeSeries->Samplig rate: " + samplingrate + " minimum gap in time: " + minimumtimegap);
|
||||
if (samplingrate == 0)
|
||||
return;
|
||||
|
||||
double[] timeline = getMillisecondsTimeline();
|
||||
AnalysisLogger.getLogger().debug("TimeSeries->filling gaps");
|
||||
double[] newvalues = SignalProcessing.fillTimeSeries(values, timeline, samplingrate, config);
|
||||
|
||||
if (newvalues.length != values.length) {
|
||||
AnalysisLogger.getLogger().debug("TimeSeries->filling also time values");
|
||||
Date[] newtimeline = SignalProcessing.fillTimeLine(timeline, samplingrate, config);
|
||||
|
||||
values = null;
|
||||
times = null;
|
||||
unsortedtimes = null;
|
||||
values = newvalues;
|
||||
times = newtimeline;
|
||||
unsortedtimes = newtimeline;
|
||||
timeLabels = new String[times.length];
|
||||
}
|
||||
|
||||
AnalysisLogger.getLogger().debug("TimeSeries->Returning values");
|
||||
timeLabels = new String[times.length];
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm", Locale.ROOT);
|
||||
for (int i = 0; i < times.length; i++) {
|
||||
timeLabels[i] = sdf.format(times[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void normalize() throws Exception {
|
||||
double max = Operations.getMax(values);
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
values[i] = values[i] / max;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
package org.gcube.dataanalysis.ecoengine.test.signalprocessing;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent;
|
||||
import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.test.regression.Regressor;
|
||||
|
||||
public class TestSimpleSignal {
|
||||
|
||||
// static AlgorithmConfiguration[] configs = {periodicSignalConfig()};//russianSignalConfig(),simpleSignalConfig()};
|
||||
static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig(), occurrencePointsSignalConfig(),hugeSignalConfig()};
|
||||
// static AlgorithmConfiguration[] configs = {russianSignalConfig()};
|
||||
// static AlgorithmConfiguration[] configs = {hugeSignalConfig()};
|
||||
// static AlgorithmConfiguration[] configs = {periodicSignalConfig()};
|
||||
public static void main(String[] args) throws Exception {
|
||||
for (int i = 0; i < configs.length; i++) {
|
||||
System.out.println("*****************TEST "+i+" *****************");
|
||||
|
||||
List<ComputationalAgent> trans = null;
|
||||
trans = TransducerersFactory.getTransducerers(configs[i]);
|
||||
trans.get(0).init();
|
||||
Regressor.process(trans.get(0));
|
||||
StatisticalType st = trans.get(0).getOutput();
|
||||
AnalysisLogger.getLogger().debug("ST:" + st);
|
||||
trans = null;
|
||||
System.out.println("*****************END TEST*****************");
|
||||
}
|
||||
}
|
||||
|
||||
public static AlgorithmConfiguration simpleSignalConfig() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setAgent("TIME_SERIES_PROCESSING");
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName", "utente");
|
||||
config.setParam("DatabasePassword", "d4science");
|
||||
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||
// vessels
|
||||
config.setParam("TimeSeriesTable", "timeseries_id4dd368bf_63fb_4d19_8e31_20ced63a477d");
|
||||
config.setParam("ValueColum", "quantity");
|
||||
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
config.setGcubeScope("/gcube");
|
||||
config.setConfigPath("./cfg");
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
public static AlgorithmConfiguration russianSignalConfig() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setAgent("TIME_SERIES_PROCESSING");
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName", "utente");
|
||||
config.setParam("DatabasePassword", "d4science");
|
||||
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||
// vessels
|
||||
config.setParam("TimeSeriesTable", "generic_ideb9efbe0_61ad_4eea_b0ee_95e64ce11b28");
|
||||
config.setParam("ValueColum", "quantity");
|
||||
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
config.setGcubeScope("/gcube");
|
||||
config.setConfigPath("./cfg");
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
public static AlgorithmConfiguration occurrencePointsSignalConfig() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setAgent("TIME_SERIES_PROCESSING");
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName", "utente");
|
||||
config.setParam("DatabasePassword", "d4science");
|
||||
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||
// vessels
|
||||
config.setParam("TimeSeriesTable", "generic_id037d302d_2ba0_4e43_b6e4_1a797bb91728");
|
||||
config.setParam("ValueColum", "speed");
|
||||
config.setParam("TimeColum", "datetime");
|
||||
config.setParam("AggregationFunction", "AVG");
|
||||
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
config.setGcubeScope("/gcube");
|
||||
config.setConfigPath("./cfg");
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
public static AlgorithmConfiguration periodicSignalConfig() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setAgent("TIME_SERIES_PROCESSING");
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName", "utente");
|
||||
config.setParam("DatabasePassword", "d4science");
|
||||
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||
// vessels
|
||||
config.setParam("TimeSeriesTable", "signalcsv");
|
||||
config.setParam("ValueColum", "signal");
|
||||
config.setParam("FrequencyResolution", "0.1");
|
||||
|
||||
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
config.setGcubeScope("/gcube");
|
||||
config.setConfigPath("./cfg");
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
public static AlgorithmConfiguration hugeSignalConfig() {
|
||||
|
||||
AlgorithmConfiguration config = new AlgorithmConfiguration();
|
||||
config.setAgent("TIME_SERIES_PROCESSING");
|
||||
config.setConfigPath("./cfg/");
|
||||
config.setPersistencePath("./");
|
||||
config.setParam("DatabaseUserName", "utente");
|
||||
config.setParam("DatabasePassword", "d4science");
|
||||
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
|
||||
config.setParam("DatabaseDriver", "org.postgresql.Driver");
|
||||
// vessels
|
||||
config.setParam("TimeSeriesTable", "generic_id634a660c_4d1a_410c_aa45_eb6e4c5afdf9");
|
||||
config.setParam("ValueColum", "quantity");
|
||||
config.setParam("TimeColum", "years");
|
||||
|
||||
|
||||
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
|
||||
|
||||
config.setGcubeScope("/gcube");
|
||||
config.setConfigPath("./cfg");
|
||||
return config;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
package org.gcube.dataanalysis.ecoengine.transducers;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
|
||||
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.InputTable;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes;
|
||||
import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates;
|
||||
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalExternalAlgorithm;
|
||||
import org.gcube.dataanalysis.ecoengine.signals.PeriodicityDetector;
|
||||
import org.gcube.dataanalysis.ecoengine.signals.SignalProcessing;
|
||||
import org.gcube.dataanalysis.ecoengine.signals.TimeSeries;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.AggregationFunctions;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils;
|
||||
import org.gcube.dataanalysis.ecoengine.utils.Tuple;
|
||||
import org.hibernate.SessionFactory;
|
||||
|
||||
public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm {
|
||||
private static String timeSeriesTable = "TimeSeriesTable";
|
||||
private static String valuesColumn= "ValueColum";
|
||||
private static String timeColumn= "TimeColum";
|
||||
private static String frequencyResolution= "FrequencyResolution";
|
||||
private static String aggregationFunction= "AggregationFunction";
|
||||
|
||||
|
||||
@Override
|
||||
public void init() throws Exception {
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "An algorithms applying signal processing to a non uniform time series. It uniformly samples the series, then extracts hidden periodicities and signal properties.";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void process() throws Exception {
|
||||
|
||||
SessionFactory dbconnection = null;
|
||||
|
||||
try{
|
||||
dbconnection = DatabaseUtils.initDBSession(config);
|
||||
String tablename = config.getParam(timeSeriesTable);
|
||||
String valuescolum = config.getParam(valuesColumn);
|
||||
// String timecolumn = "datetime";//"time";
|
||||
String timecolumn = config.getParam(timeColumn);
|
||||
String aggregationFunc= config.getParam(aggregationFunction);
|
||||
String frequencyRes= config.getParam(frequencyResolution);
|
||||
float frequencyResDouble = 1;
|
||||
if (timecolumn==null)
|
||||
timecolumn = "time";
|
||||
if (aggregationFunc==null)
|
||||
aggregationFunc="SUM";
|
||||
if (frequencyRes!=null)
|
||||
{ try{
|
||||
frequencyResDouble=Float.parseFloat(frequencyRes);
|
||||
}catch(Exception e){
|
||||
}
|
||||
}
|
||||
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Table Name: "+tablename);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Time Column: "+timecolumn);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Values Column: "+valuescolum);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Aggregation: "+aggregationFunc);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Frequency Resolution: "+frequencyRes);
|
||||
|
||||
// String query = "select * from (select "+valuescolum+","+timecolumn+" from "+tablename+") as a";
|
||||
String query = "select * from (select "+aggregationFunc+"("+valuescolum+"),"+timecolumn+" from "+tablename+" group by "+timecolumn+") as a";
|
||||
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Query to execute: "+query);
|
||||
List<Object> results = DatabaseFactory.executeSQLQuery(query, dbconnection);
|
||||
if (results==null || results.size()==0)
|
||||
throw new Exception("Error in retrieving values from the table: no time series found");
|
||||
//build signal
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Building signal");
|
||||
List<Tuple<String>> signal = new ArrayList<Tuple<String>>();
|
||||
int sizesignal = 0;
|
||||
for (Object row:results){
|
||||
Object[] srow = (Object[]) row;
|
||||
String value =""+srow[0];
|
||||
String time=""+srow[1];
|
||||
signal.add(new Tuple<String>(time,value));
|
||||
sizesignal++;
|
||||
}
|
||||
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Signal built with success. Size: "+sizesignal);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Building Time Series");
|
||||
TimeSeries ts = TimeSeries.buildFromSignal(signal, config);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniformly sampling the signal");
|
||||
SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Time Series", "HH:mm:ss MM-dd-yy");
|
||||
ts.convertToUniformSignal(0);
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniform sampling finished");
|
||||
//spectrum and signal processing
|
||||
SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Unif Samp Time Series", "HH:mm:ss MM-dd-yy");
|
||||
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Detecting periodicity");
|
||||
PeriodicityDetector pd = new PeriodicityDetector();
|
||||
double F = pd.detectFrequency(ts.getValues(),1,0.01f,0.5f,frequencyResDouble,true);
|
||||
|
||||
AnalysisLogger.getLogger().debug("Detected Frequency: "+F+" indecision ["+pd.lowermeanF+" , "+pd.uppermeanF+"]");
|
||||
AnalysisLogger.getLogger().debug("Detected Period: "+pd.meanPeriod+" indecision ["+pd.lowermeanPeriod+" , "+pd.uppermeanPeriod+"]");
|
||||
AnalysisLogger.getLogger().debug("Detected Periodicity Strength: "+pd.periodicityStrength+" ("+pd.getPeriodicityStregthInterpretation()+")");
|
||||
AnalysisLogger.getLogger().debug("Periodicity inside this samples range: ["+pd.startPeriodTime+";"+pd.endPeriodTime+"]");
|
||||
|
||||
AnalysisLogger.getLogger().debug("Maximum Frequency in the Spectrogram "+MathFunctions.roundDecimal(pd.maxFrequency,2));
|
||||
AnalysisLogger.getLogger().debug("Minimum Frequency in the Spectrogram "+MathFunctions.roundDecimal(pd.minFrequency,2));
|
||||
|
||||
// Image spectrogramImage = SignalProcessing.renderSignalSpectrogram2(pd.currentspectrum);
|
||||
|
||||
|
||||
}catch(Exception e){
|
||||
e.printStackTrace();
|
||||
throw e;
|
||||
}finally{
|
||||
if (dbconnection!=null)
|
||||
dbconnection.close();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setInputParameters() {
|
||||
//the time series table
|
||||
List<TableTemplates> templates = new ArrayList<TableTemplates>();
|
||||
templates.add(TableTemplates.TIMESERIES);
|
||||
InputTable p = new InputTable(templates, timeSeriesTable, "The table containing the time series", "timeseries");
|
||||
inputs.add(p);
|
||||
ColumnType p1 = new ColumnType(timeSeriesTable, valuesColumn, "The column containing the values of the time series", "values", false);
|
||||
inputs.add(p1);
|
||||
addDoubleInput(frequencyResolution, "The precision in detecting the period. The lower this number the lower the number of samples used at each step in the Spectrogram. Reducing this, the spectrogram will be finer and sharper, but you should tune it. Too few samples will make the Spectrogram noisy.", "1");
|
||||
addEnumerateInput(AggregationFunctions.values(), aggregationFunction, "Function to apply to samples with the same time instant", AggregationFunctions.SUM.name());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package org.gcube.dataanalysis.ecoengine.utils;
|
||||
|
||||
public enum AggregationFunctions {
|
||||
|
||||
SUM,
|
||||
AVG
|
||||
}
|
||||
Loading…
Reference in New Issue