adjustments to time series analysis

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine@95036 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gianpaolo Coro 2014-04-22 16:13:26 +00:00
parent 1e5ac19f5b
commit e5629d2300
5 changed files with 195 additions and 149 deletions

View File

@ -178,4 +178,14 @@ public class DateGuesser {
return YEAR;
}
public static boolean isJavaDateOrigin(Date date){
Calendar c = Calendar.getInstance();
c.setTime(date);
if ((c.get(Calendar.DAY_OF_MONTH)==1) && (c.get(Calendar.MONTH) ==0 ) && (c.get(Calendar.YEAR) == 1970))
return true;
else
return false;
}
}

View File

@ -146,7 +146,7 @@ public class PeriodicityDetector {
AnalysisLogger.getLogger().debug("Number of frequency peaks " + maxfrequencies.length);
// take the longest stable sequence of frequencies
SignalConverter signalconverter = new SignalConverter();
maxfrequencies = signalconverter.takeLongestStableTract(maxfrequencies, 0.5);
maxfrequencies = signalconverter.takeLongestStableTract(maxfrequencies, 0.01);
if (maxfrequencies == null)
return 0;
@ -170,7 +170,11 @@ public class PeriodicityDetector {
// reconstruct the F
double meanF = MathFunctions.mean(maxfrequencies);
if ((meanF <= minPossibleFreq) || (meanF >= maxPossibleFreq)) {
//we consider a complete cycle
double possibleperiod = 2d/meanF;
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Frequency "+meanF);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Periodicity "+possibleperiod);
if ((meanF <= minPossibleFreq) || (meanF >= maxPossibleFreq) || (possibleperiod==0) || (possibleperiod>(endPeriodTime-startPeriodTime))) {
meanF=0;
this.meanF = 0;
this.lowermeanF = 0;
@ -190,9 +194,9 @@ public class PeriodicityDetector {
this.lowermeanF = Math.max(meanF - error, minPossibleFreq);
this.uppermeanF = Math.min(meanF + error, maxFrequency);
this.meanPeriod = 1d / meanF;
this.lowermeanPeriod = 1d / lowermeanF;
this.uppermeanPeriod = 1d / uppermeanF;
this.meanPeriod = 2d / meanF;
this.lowermeanPeriod = 2d / lowermeanF;
this.uppermeanPeriod = 2d / uppermeanF;
}
return meanF;
}

View File

@ -12,6 +12,7 @@ import marytts.signalproc.window.Window;
import org.gcube.contentmanagement.graphtools.data.BigSamplesTable;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.transducers.TimeSeriesAnalysis;
import com.rapidminer.example.Attribute;
import com.rapidminer.example.Example;
@ -19,36 +20,19 @@ import com.rapidminer.example.ExampleSet;
import com.rapidminer.example.table.MemoryExampleTable;
/**
* includes tools for basic signal transformations:
* delta + double delta
* center frequency
* cepstral coefficients calculation
* spectrum frequency cut
* transformation to and from Rapid Miner Example Set
* filterbanks
* fequency to mel
* frequency to index in fft
* sinusoid signal generation
* inverse mel
* log10
* mel filterbanks
* sample to time and time to sample
* signal timeline generation
* index to time in spectrogram
* spectrogram calculation and display
* time to index in spectrogram
* includes tools for basic signal transformations: delta + double delta center frequency cepstral coefficients calculation spectrum frequency cut transformation to and from Rapid Miner Example Set filterbanks fequency to mel frequency to index in fft sinusoid signal generation inverse mel log10 mel filterbanks sample to time and time to sample signal timeline generation index to time in spectrogram spectrogram calculation and display time to index in spectrogram
*
* @author coro
*
*/
public class SignalConverter {
public static double[][] addDeltaDouble(double[][] features) throws Exception{
public static double[][] addDeltaDouble(double[][] features) throws Exception {
int vectorL = features[0].length;
double[][] delta = new double[features.length][features[0].length*3];
double[][] delta = new double[features.length][features[0].length * 3];
for (int k=0;k<features.length;k++){
for (int g=0;g<vectorL;g++){
for (int k = 0; k < features.length; k++) {
for (int g = 0; g < vectorL; g++) {
delta[k][g] = features[k][g];
}
}
@ -83,8 +67,8 @@ public class SignalConverter {
public static BufferedImage createImage(JPanel panel, int w, int h) {
// int w = panel.getWidth();
// int h = panel.getHeight();
// int w = panel.getWidth();
// int h = panel.getHeight();
BufferedImage bi = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB);
Graphics2D g = bi.createGraphics();
panel.paint(g);
@ -133,18 +117,18 @@ public class SignalConverter {
}
public static int[] fftBinIndices(double samplingRate, int frameSize, int numMelFilters, int numFequencies, float lowerFilterFreq, float upperFilterFreq) {
int cbin[] = new int[numFequencies+2];
AnalysisLogger.getLogger().debug("New Filter banks: "+numFequencies);
int cbin[] = new int[numFequencies + 2];
AnalysisLogger.getLogger().debug("New Filter banks: " + numFequencies);
cbin[0] = (int) Math.round(lowerFilterFreq / samplingRate * frameSize);
cbin[cbin.length - 1] = frequencyIndex(upperFilterFreq, frameSize, (float)samplingRate);
AnalysisLogger.getLogger().debug("F0: "+lowerFilterFreq);
cbin[cbin.length - 1] = frequencyIndex(upperFilterFreq, frameSize, (float) samplingRate);
AnalysisLogger.getLogger().debug("F0: " + lowerFilterFreq);
for (int i = 1; i <= numFequencies; i++) {
double fc = centerFreq(i, samplingRate, lowerFilterFreq, numMelFilters);
AnalysisLogger.getLogger().debug("F"+(i)+": "+fc);
AnalysisLogger.getLogger().debug("F" + (i) + ": " + fc);
cbin[i] = (int) Math.round(fc / samplingRate * frameSize);
}
AnalysisLogger.getLogger().debug("F"+(cbin.length - 1)+": "+upperFilterFreq);
AnalysisLogger.getLogger().debug("F" + (cbin.length - 1) + ": " + upperFilterFreq);
return cbin;
}
@ -168,13 +152,12 @@ public class SignalConverter {
return 2595 * log10(1 + freq / 700);
}
public static int frequencyIndex(float frequency, int fftSize, float samplingRate) {
return Math.round(frequency * fftSize / samplingRate);
}
public static double[] generateSinSignal(int signalLength, float timeShift, float frequency) {
//final float frequency = 0.3f;// 1f;
// final float frequency = 0.3f;// 1f;
double samples[] = new double[signalLength];
float time = 0;
@ -191,7 +174,6 @@ public class SignalConverter {
return 700 * (temp);
}
public static double log10(double value) {
return Math.log(value) / Math.log(10);
}
@ -221,18 +203,18 @@ public class SignalConverter {
return fbank;
}
public static int recalculateMaxMelFilters(double samplingRate, int numMelFilters, float lowerFilterFreq, float maxFilterFreq){
int bestIndex=1;
public static int recalculateMaxMelFilters(double samplingRate, int numMelFilters, float lowerFilterFreq, float maxFilterFreq) {
int bestIndex = 1;
for (int i = 1; i <= numMelFilters; i++) {
double fc = centerFreq(i, samplingRate, lowerFilterFreq, numMelFilters);
AnalysisLogger.getLogger().debug("fc "+fc);
if (fc>maxFilterFreq){
bestIndex=i;
AnalysisLogger.getLogger().debug("fc " + fc);
if (fc > maxFilterFreq) {
bestIndex = i;
break;
}
}
return bestIndex-1;
return bestIndex - 1;
}
public static double sample2Time(int sample, int sampleRate) {
@ -254,7 +236,7 @@ public class SignalConverter {
return linearTime / windowShiftTime;
}
public static ExampleSet signal2ExampleSet(double[] signal){
public static ExampleSet signal2ExampleSet(double[] signal) {
BigSamplesTable samples = new BigSamplesTable();
for (int k = 0; k < signal.length; k++) {
samples.addSampleRow("" + k, signal[k]);
@ -269,13 +251,10 @@ public class SignalConverter {
if (display) {
spec.showInJFrame(name, true, true);
/*
* save spectrograms to files BufferedImage image =
* createImage(spec);
* ImageIO.write(ImageTools.toBufferedImage(image), "png", new
* File(name+".png"));
* save spectrograms to files BufferedImage image = createImage(spec); ImageIO.write(ImageTools.toBufferedImage(image), "png", new File(name+".png"));
*/
// Thread.sleep(2000);
// createImage(spec);
// createImage(spec);
}
return spectrum;
}
@ -285,7 +264,7 @@ public class SignalConverter {
SpectrogramCustom spec = new SpectrogramCustom(signal, samplingRate, Window.get(Window.HAMMING, frameslength), windowshift, frameslength, 640, 480);
spec.spectra = new ArrayList<double[]>();
for (int i=0;i<spectrum.length;i++){
for (int i = 0; i < spectrum.length; i++) {
spec.spectra.add(spectrum[i]);
}
spec.showInJFrame(name, true, true);
@ -305,89 +284,98 @@ public class SignalConverter {
}
public double[] averagepower;
public double[] takeMaxFrequenciesInSpectrogram(double[][] spectrogram, int samplingRate,int windowSamples, float minfreq) {
public double[] takeMaxFrequenciesInSpectrogram(double[][] spectrogram, int samplingRate, int windowSamples, float minfreq) {
double[] maxs = new double[spectrogram.length];
averagepower = new double[spectrogram.length];
int j = 0;
// SignalProcessing.displaySignalWithGenericTime(spectrogram[0], 0, 1, "spectrum");
if (TimeSeriesAnalysis.display)
SignalProcessing.displaySignalWithGenericTime(spectrogram[0], 0, 1, "spectrum");
double tolerance = 0.05;
for (double[] slice : spectrogram) {
int bestidx = 0;
double max = -Double.MAX_VALUE;
double min = Double.MAX_VALUE;
for (int k = 0; k < slice.length; k++) {
//take the best frequency over the first minimum in the spectrum
boolean overfirstmin = false;
for (int k = 1; k < slice.length; k++) {
double ele = slice[k];
if (ele > (max+(Math.abs(max)*0.1))) {
if (!overfirstmin && (slice[k]>slice[k-1])){
AnalysisLogger.getLogger().debug("First minimum in spectrum is at idx "+k);
overfirstmin=true;
}
if (overfirstmin) {
if (ele > (max + (Math.abs(max) * tolerance))) {
AnalysisLogger.getLogger().debug(">max up:"+ele +">" +(max + (Math.abs(max) * tolerance))+" at idx "+k);
max = ele;
bestidx = k;
// AnalysisLogger.getLogger().debug(">max up:"+ele +">" +(max-max*0.1));
}
if (ele<(min-(Math.abs(min)*0.1))){
if (ele < (min - (Math.abs(min) * tolerance))) {
min = ele;
}
}
}
// maxs[j] = spectrogram[j][bestidx];
// maxs[j]=bestidx;
// maxs[j] = spectrogram[j][bestidx];
// maxs[j]=bestidx;
int minFidx = SignalConverter.frequencyIndex(minfreq, windowSamples, samplingRate);
// System.out.println("min f idx: "+minFidx);
maxs[j] = spectrumIdx2Frequency(minFidx+bestidx,samplingRate,windowSamples);
// System.out.println("min f idx: "+minFidx);
maxs[j] = spectrumIdx2Frequency(minFidx + bestidx, samplingRate, windowSamples);
double mean = org.gcube.contentmanagement.graphtools.utils.MathFunctions.mean(slice);
AnalysisLogger.getLogger().debug("max freq in spec: "+maxs[j]);
if (min==Double.MAX_VALUE){
AnalysisLogger.getLogger().debug("max freq in spec: " + maxs[j]+" index "+minFidx + bestidx);
if (min == Double.MAX_VALUE) {
min = max;
}
if (max==-Double.MAX_VALUE){
averagepower[j]=0;
}
else{
max=max-min;
mean = mean-min;
if (max ==0)
averagepower[j]=0;
if (max == -Double.MAX_VALUE) {
averagepower[j] = 0;
} else {
max = max - min;
mean = mean - min;
if (max == 0)
averagepower[j] = 0;
else
averagepower[j]=Math.abs((max-mean)/max);
averagepower[j] = Math.abs((max - mean) / max);
}
AnalysisLogger.getLogger().debug("max power : "+max+" min power: "+min+" mean "+mean+" power "+averagepower[j]);
AnalysisLogger.getLogger().debug("max power : " + max + " min power: " + min + " mean " + mean + " power " + averagepower[j]);
j++;
}
return maxs;
}
public int startStableTractIdx=-1;
public int endStableTractIdx=-1;
public int startStableTractIdx = -1;
public int endStableTractIdx = -1;
public double[] takeLongestStableTract(double[] signal, double valuedifftoleranceperc) {
ArrayList<int[]> pairs = new ArrayList<int[]>();
int idx1 = -1;
int[] pair = null;
//analyze the signal
// analyze the signal
for (int i = 1; i < signal.length; i++) {
//if there is not current range create it
// if there is not current range create it
if (idx1 == -1) {
idx1 = 1;
pair = new int[2];
pair[0] = i - 1;
pair[1] = i - 1;
}
//if the current sample is similar to the previous, enlarge the range
if (Math.abs(signal[i] - signal[i - 1])/Math.max(signal[i],signal[i - 1]) <= valuedifftoleranceperc)
// if the current sample is similar to the previous, enlarge the range
if (Math.abs(signal[i] - signal[i - 1]) / Math.max(signal[i], signal[i - 1]) <= valuedifftoleranceperc)
pair[1] = i;
//otherwise add the couple and reset
// otherwise add the couple and reset
else {
idx1 = -1;
pairs.add(pair);
}
}
//if the last couple was reset, add the last interval
// if the last couple was reset, add the last interval
if (idx1 > -1)
pairs.add(pair);
//find the longest pair
// find the longest pair
int best = 0;
int maxsize = 0;
int k = 0;
@ -400,15 +388,15 @@ public class SignalConverter {
k++;
}
//take the longest range
if (pairs.size()==0){
pairs.add(new int[] {0,1});
// take the longest range
if (pairs.size() == 0) {
pairs.add(new int[] { 0, 1 });
}
int[] bestcouple = pairs.get(best);
//take the related slice of signal
// take the related slice of signal
double[] subsignal = new double[bestcouple[1] - bestcouple[0]];
AnalysisLogger.getLogger().debug("Longest range: from "+bestcouple[0]+" to "+bestcouple[1]);
AnalysisLogger.getLogger().debug("Longest range: from " + bestcouple[0] + " to " + bestcouple[1]);
startStableTractIdx = bestcouple[0];
endStableTractIdx = bestcouple[1];
@ -421,13 +409,12 @@ public class SignalConverter {
return subsignal;
}
public static float spectrumIdx2Frequency(int idx,int samplingRate,int windowsSizeSamples){
return ((float)idx*samplingRate)/(float)(windowsSizeSamples-1);
public static float spectrumIdx2Frequency(int idx, int samplingRate, int windowsSizeSamples) {
return ((float) idx * samplingRate) / (2f*(float) (windowsSizeSamples - 1));
}
public static int spectrumFreq2Idx(float freq, int samplingRate,int windowsSizeSamples){
return Math.round((windowsSizeSamples-1)*freq/samplingRate);
public static int spectrumFreq2Idx(float freq, int samplingRate, int windowsSizeSamples) {
return Math.round((windowsSizeSamples - 1) * 2f *freq / samplingRate);
}
}

View File

@ -13,13 +13,18 @@ public class TestSimpleSignal {
// static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig(), occurrencePointsSignalConfig(),hugeSignalConfig()};
//static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig()};
//static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig()};
// static AlgorithmConfiguration[] configs = {NAFOSignalConfig()};
// static AlgorithmConfiguration[] configs = {largeCustomSignalConfig()};
// static AlgorithmConfiguration[] configs = {temperatureSignalConfig()};
// static AlgorithmConfiguration[] configs = {periodicSignalConfig()};
static AlgorithmConfiguration[] configs = {simpleSignalConfig()};
// static AlgorithmConfiguration[] configs = {sawSignalConfig()};
static AlgorithmConfiguration[] configs = {earthquakesSignalConfig()};
// static AlgorithmConfiguration[] configs = {temperatureSignalConfig()};
// static AlgorithmConfiguration[] configs = {russianSignalConfig()};
// static AlgorithmConfiguration[] configs = {largeCustomSignalConfig()};
// static AlgorithmConfiguration[] configs = {occurrencePointsSignalConfig()};
// static AlgorithmConfiguration[] configs = {hugeSignalConfig()};
public static void main(String[] args) throws Exception {
int wLength = (int) Math.pow(2, 1);
@ -127,9 +132,9 @@ public class TestSimpleSignal {
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "signalcsv");
config.setParam("ValueColum", "signal");
config.setParam("FrequencyResolution", "1");
config.setParam("TimeSeriesTable", "timeseries_ide814eb07_c13b_41b3_a240_aa99446db831");
config.setParam("ValueColum", "quantity");
config.setParam("FrequencyResolution", "0.01");
config.setParam("SSA_Window_in_Samples", "20");
config.setParam("SSA_EigenvaluesThreshold", "0.7");
config.setParam("SSA_Points_to_Forecast", "10");
@ -284,5 +289,33 @@ public class TestSimpleSignal {
public static AlgorithmConfiguration temperatureSignalConfig() {
AlgorithmConfiguration config = new AlgorithmConfiguration();
config.setAgent("TIME_SERIES_PROCESSING");
config.setConfigPath("./cfg/");
config.setPersistencePath("./");
config.setParam("DatabaseUserName", "utente");
config.setParam("DatabasePassword", "d4science");
config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb");
config.setParam("DatabaseDriver", "org.postgresql.Driver");
// vessels
config.setParam("TimeSeriesTable", "timeseries_id08b3abb9_c7b0_4b82_8117_64b69055416f");
config.setParam("ValueColum", "fvalue");
config.setParam("FrequencyResolution", "0.01");
config.setParam("SSA_Window_in_Samples", "10");
config.setParam("SSA_EigenvaluesThreshold", "0.7");
config.setParam("SSA_Points_to_Forecast", "10");
AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile);
config.setGcubeScope("/gcube");
config.setConfigPath("./cfg");
return config;
}
}

View File

@ -10,6 +10,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import org.gcube.contentmanagement.graphtools.utils.DateGuesser;
import org.gcube.contentmanagement.graphtools.utils.MathFunctions;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType;
@ -48,7 +49,7 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm {
private Image forecastsignalImg = null;
private Image eigenValuesImg=null;
private File outputfilename=null;
private static boolean display = false;
public static boolean display = true;
private static int maxpoints = 10000;
@Override
public void init() throws Exception {
@ -129,8 +130,16 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm {
TimeSeries ts = TimeSeries.buildFromSignal(signal, config);
String timepattern = ts.getTimepattern();
String chartpattern = "HH:mm:ss MM-dd-yy";
if (timepattern.equals("s"))
if (timepattern.equals("s") ||
(DateGuesser.isJavaDateOrigin(ts.getTime()[0]) &&
DateGuesser.isJavaDateOrigin(ts.getTime()[ts.getTime().length-1]))
)
{
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Changing chart pattern to Seconds!");
chartpattern = "HH:mm:ss:SS";
}
else
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Chart pattern remains "+chartpattern);
AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniformly sampling the signal");
if (display)
@ -286,4 +295,7 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm {
}
}