diff --git a/src/main/java/org/gcube/contentmanagement/graphtools/utils/DateGuesser.java b/src/main/java/org/gcube/contentmanagement/graphtools/utils/DateGuesser.java index 61f1ab8..9bb1b38 100644 --- a/src/main/java/org/gcube/contentmanagement/graphtools/utils/DateGuesser.java +++ b/src/main/java/org/gcube/contentmanagement/graphtools/utils/DateGuesser.java @@ -177,5 +177,15 @@ public class DateGuesser { return YEAR; } + + + public static boolean isJavaDateOrigin(Date date){ + Calendar c = Calendar.getInstance(); + c.setTime(date); + if ((c.get(Calendar.DAY_OF_MONTH)==1) && (c.get(Calendar.MONTH) ==0 ) && (c.get(Calendar.YEAR) == 1970)) + return true; + else + return false; + } } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java index 02fef86..fa2cf81 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/PeriodicityDetector.java @@ -146,7 +146,7 @@ public class PeriodicityDetector { AnalysisLogger.getLogger().debug("Number of frequency peaks " + maxfrequencies.length); // take the longest stable sequence of frequencies SignalConverter signalconverter = new SignalConverter(); - maxfrequencies = signalconverter.takeLongestStableTract(maxfrequencies, 0.5); + maxfrequencies = signalconverter.takeLongestStableTract(maxfrequencies, 0.01); if (maxfrequencies == null) return 0; @@ -170,7 +170,11 @@ public class PeriodicityDetector { // reconstruct the F double meanF = MathFunctions.mean(maxfrequencies); - if ((meanF <= minPossibleFreq) || (meanF >= maxPossibleFreq)) { + //we consider a complete cycle + double possibleperiod = 2d/meanF; + AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Frequency "+meanF); + AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Periodicity "+possibleperiod); + if ((meanF <= minPossibleFreq) || (meanF >= maxPossibleFreq) || (possibleperiod==0) || (possibleperiod>(endPeriodTime-startPeriodTime))) { meanF=0; this.meanF = 0; this.lowermeanF = 0; @@ -190,9 +194,9 @@ public class PeriodicityDetector { this.lowermeanF = Math.max(meanF - error, minPossibleFreq); this.uppermeanF = Math.min(meanF + error, maxFrequency); - this.meanPeriod = 1d / meanF; - this.lowermeanPeriod = 1d / lowermeanF; - this.uppermeanPeriod = 1d / uppermeanF; + this.meanPeriod = 2d / meanF; + this.lowermeanPeriod = 2d / lowermeanF; + this.uppermeanPeriod = 2d / uppermeanF; } return meanF; } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java index a213ed7..08be05a 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/signals/SignalConverter.java @@ -12,6 +12,7 @@ import marytts.signalproc.window.Window; import org.gcube.contentmanagement.graphtools.data.BigSamplesTable; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.transducers.TimeSeriesAnalysis; import com.rapidminer.example.Attribute; import com.rapidminer.example.Example; @@ -19,46 +20,29 @@ import com.rapidminer.example.ExampleSet; import com.rapidminer.example.table.MemoryExampleTable; /** - * includes tools for basic signal transformations: - * delta + double delta - * center frequency - * cepstral coefficients calculation - * spectrum frequency cut - * transformation to and from Rapid Miner Example Set - * filterbanks - * fequency to mel - * frequency to index in fft - * sinusoid signal generation - * inverse mel - * log10 - * mel filterbanks - * sample to time and time to sample - * signal timeline generation - * index to time in spectrogram - * spectrogram calculation and display - * time to index in spectrogram - * + * includes tools for basic signal transformations: delta + double delta center frequency cepstral coefficients calculation spectrum frequency cut transformation to and from Rapid Miner Example Set filterbanks fequency to mel frequency to index in fft sinusoid signal generation inverse mel log10 mel filterbanks sample to time and time to sample signal timeline generation index to time in spectrogram spectrogram calculation and display time to index in spectrogram + * * @author coro - * + * */ public class SignalConverter { - public static double[][] addDeltaDouble(double[][] features) throws Exception{ + public static double[][] addDeltaDouble(double[][] features) throws Exception { int vectorL = features[0].length; - double[][] delta = new double[features.length][features[0].length*3]; - - for (int k=0;kmaxFilterFreq){ - bestIndex=i; + AnalysisLogger.getLogger().debug("fc " + fc); + if (fc > maxFilterFreq) { + bestIndex = i; break; } } - return bestIndex-1; + return bestIndex - 1; } - + public static double sample2Time(int sample, int sampleRate) { return (double) sample / (double) sampleRate; } - + public static double[] signalTimeLine(int signalLength, double samplingRate) { double time[] = new double[signalLength]; Arrays.fill(time, Double.NaN); @@ -249,12 +231,12 @@ public class SignalConverter { AnalysisLogger.getLogger().debug("time " + time[signalLength - 1] * samplingRate + " vs " + signalLength); return time; } - + public static float spectrumTime(float linearTime, float windowShiftTime) { return linearTime / windowShiftTime; } - public static ExampleSet signal2ExampleSet(double[] signal){ + public static ExampleSet signal2ExampleSet(double[] signal) { BigSamplesTable samples = new BigSamplesTable(); for (int k = 0; k < signal.length; k++) { samples.addSampleRow("" + k, signal[k]); @@ -262,40 +244,37 @@ public class SignalConverter { AnalysisLogger.getLogger().debug("Example Set Created"); return samples.generateExampleSet(); } - + public static double[][] spectrogram(String name, double[] signal, int samplingRate, int windowshift, int frameslength, boolean display) throws Exception { SpectrogramCustom spec = new SpectrogramCustom(signal, samplingRate, Window.get(Window.HAMMING, frameslength), windowshift, frameslength, 640, 480); double[][] spectrum = spec.spectra.toArray(new double[spec.spectra.size()][]); if (display) { spec.showInJFrame(name, true, true); /* - * save spectrograms to files BufferedImage image = - * createImage(spec); - * ImageIO.write(ImageTools.toBufferedImage(image), "png", new - * File(name+".png")); + * save spectrograms to files BufferedImage image = createImage(spec); ImageIO.write(ImageTools.toBufferedImage(image), "png", new File(name+".png")); */ // Thread.sleep(2000); -// createImage(spec); + // createImage(spec); } return spectrum; } - + public static void displaySpectrogram(double[][] spectrum, double[] signal, String name, int samplingRate, int windowshift, int frameslength) throws Exception { - + SpectrogramCustom spec = new SpectrogramCustom(signal, samplingRate, Window.get(Window.HAMMING, frameslength), windowshift, frameslength, 640, 480); - + spec.spectra = new ArrayList(); - for (int i=0;i (max+(Math.abs(max)*0.1))) { - max = ele; - bestidx = k; -// AnalysisLogger.getLogger().debug(">max up:"+ele +">" +(max-max*0.1)); + if (!overfirstmin && (slice[k]>slice[k-1])){ + AnalysisLogger.getLogger().debug("First minimum in spectrum is at idx "+k); + overfirstmin=true; } - if (ele<(min-(Math.abs(min)*0.1))){ - min = ele; + if (overfirstmin) { + if (ele > (max + (Math.abs(max) * tolerance))) { + AnalysisLogger.getLogger().debug(">max up:"+ele +">" +(max + (Math.abs(max) * tolerance))+" at idx "+k); + max = ele; + bestidx = k; + } + if (ele < (min - (Math.abs(min) * tolerance))) { + min = ele; + } } } -// maxs[j] = spectrogram[j][bestidx]; -// maxs[j]=bestidx; - - + // maxs[j] = spectrogram[j][bestidx]; + // maxs[j]=bestidx; + int minFidx = SignalConverter.frequencyIndex(minfreq, windowSamples, samplingRate); -// System.out.println("min f idx: "+minFidx); - maxs[j] = spectrumIdx2Frequency(minFidx+bestidx,samplingRate,windowSamples); + // System.out.println("min f idx: "+minFidx); + maxs[j] = spectrumIdx2Frequency(minFidx + bestidx, samplingRate, windowSamples); double mean = org.gcube.contentmanagement.graphtools.utils.MathFunctions.mean(slice); - AnalysisLogger.getLogger().debug("max freq in spec: "+maxs[j]); - if (min==Double.MAX_VALUE){ - min = max; + AnalysisLogger.getLogger().debug("max freq in spec: " + maxs[j]+" index "+minFidx + bestidx); + if (min == Double.MAX_VALUE) { + min = max; } - if (max==-Double.MAX_VALUE){ - averagepower[j]=0; - } - else{ - max=max-min; - mean = mean-min; - if (max ==0) - averagepower[j]=0; + if (max == -Double.MAX_VALUE) { + averagepower[j] = 0; + } else { + max = max - min; + mean = mean - min; + if (max == 0) + averagepower[j] = 0; else - averagepower[j]=Math.abs((max-mean)/max); + averagepower[j] = Math.abs((max - mean) / max); } - - AnalysisLogger.getLogger().debug("max power : "+max+" min power: "+min+" mean "+mean+" power "+averagepower[j]); + + AnalysisLogger.getLogger().debug("max power : " + max + " min power: " + min + " mean " + mean + " power " + averagepower[j]); j++; } return maxs; } - public int startStableTractIdx=-1; - public int endStableTractIdx=-1; - + public int startStableTractIdx = -1; + public int endStableTractIdx = -1; + public double[] takeLongestStableTract(double[] signal, double valuedifftoleranceperc) { ArrayList pairs = new ArrayList(); int idx1 = -1; int[] pair = null; - //analyze the signal + // analyze the signal for (int i = 1; i < signal.length; i++) { - //if there is not current range create it + // if there is not current range create it if (idx1 == -1) { idx1 = 1; pair = new int[2]; pair[0] = i - 1; pair[1] = i - 1; } - //if the current sample is similar to the previous, enlarge the range - if (Math.abs(signal[i] - signal[i - 1])/Math.max(signal[i],signal[i - 1]) <= valuedifftoleranceperc) + // if the current sample is similar to the previous, enlarge the range + if (Math.abs(signal[i] - signal[i - 1]) / Math.max(signal[i], signal[i - 1]) <= valuedifftoleranceperc) pair[1] = i; - //otherwise add the couple and reset + // otherwise add the couple and reset else { idx1 = -1; pairs.add(pair); } } - //if the last couple was reset, add the last interval + // if the last couple was reset, add the last interval if (idx1 > -1) pairs.add(pair); - - //find the longest pair + + // find the longest pair int best = 0; int maxsize = 0; int k = 0; @@ -399,19 +387,19 @@ public class SignalConverter { } k++; } - - //take the longest range - if (pairs.size()==0){ - pairs.add(new int[] {0,1}); + + // take the longest range + if (pairs.size() == 0) { + pairs.add(new int[] { 0, 1 }); } - + int[] bestcouple = pairs.get(best); - //take the related slice of signal + // take the related slice of signal double[] subsignal = new double[bestcouple[1] - bestcouple[0]]; - AnalysisLogger.getLogger().debug("Longest range: from "+bestcouple[0]+" to "+bestcouple[1]); + AnalysisLogger.getLogger().debug("Longest range: from " + bestcouple[0] + " to " + bestcouple[1]); startStableTractIdx = bestcouple[0]; endStableTractIdx = bestcouple[1]; - + int l = 0; for (int i = bestcouple[0]; i < bestcouple[1]; i++) { subsignal[l] = signal[i]; @@ -421,13 +409,12 @@ public class SignalConverter { return subsignal; } - public static float spectrumIdx2Frequency(int idx,int samplingRate,int windowsSizeSamples){ - return ((float)idx*samplingRate)/(float)(windowsSizeSamples-1); + public static float spectrumIdx2Frequency(int idx, int samplingRate, int windowsSizeSamples) { + return ((float) idx * samplingRate) / (2f*(float) (windowsSizeSamples - 1)); } - - public static int spectrumFreq2Idx(float freq, int samplingRate,int windowsSizeSamples){ - return Math.round((windowsSizeSamples-1)*freq/samplingRate); + + public static int spectrumFreq2Idx(float freq, int samplingRate, int windowsSizeSamples) { + return Math.round((windowsSizeSamples - 1) * 2f *freq / samplingRate); } - - + } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/test/signalprocessing/TestSimpleSignal.java b/src/main/java/org/gcube/dataanalysis/ecoengine/test/signalprocessing/TestSimpleSignal.java index e507aa7..c0f584f 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/test/signalprocessing/TestSimpleSignal.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/test/signalprocessing/TestSimpleSignal.java @@ -13,13 +13,18 @@ public class TestSimpleSignal { // static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig(), occurrencePointsSignalConfig(),hugeSignalConfig()}; - //static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig()}; +//static AlgorithmConfiguration[] configs = {periodicSignalConfig(), russianSignalConfig(),simpleSignalConfig()}; // static AlgorithmConfiguration[] configs = {NAFOSignalConfig()}; // static AlgorithmConfiguration[] configs = {largeCustomSignalConfig()}; +// static AlgorithmConfiguration[] configs = {temperatureSignalConfig()}; +// static AlgorithmConfiguration[] configs = {periodicSignalConfig()}; + static AlgorithmConfiguration[] configs = {simpleSignalConfig()}; // static AlgorithmConfiguration[] configs = {sawSignalConfig()}; - static AlgorithmConfiguration[] configs = {earthquakesSignalConfig()}; - - +// static AlgorithmConfiguration[] configs = {temperatureSignalConfig()}; +// static AlgorithmConfiguration[] configs = {russianSignalConfig()}; +// static AlgorithmConfiguration[] configs = {largeCustomSignalConfig()}; +// static AlgorithmConfiguration[] configs = {occurrencePointsSignalConfig()}; +// static AlgorithmConfiguration[] configs = {hugeSignalConfig()}; public static void main(String[] args) throws Exception { int wLength = (int) Math.pow(2, 1); @@ -127,9 +132,9 @@ public class TestSimpleSignal { config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); config.setParam("DatabaseDriver", "org.postgresql.Driver"); // vessels - config.setParam("TimeSeriesTable", "signalcsv"); - config.setParam("ValueColum", "signal"); - config.setParam("FrequencyResolution", "1"); + config.setParam("TimeSeriesTable", "timeseries_ide814eb07_c13b_41b3_a240_aa99446db831"); + config.setParam("ValueColum", "quantity"); + config.setParam("FrequencyResolution", "0.01"); config.setParam("SSA_Window_in_Samples", "20"); config.setParam("SSA_EigenvaluesThreshold", "0.7"); config.setParam("SSA_Points_to_Forecast", "10"); @@ -284,5 +289,33 @@ public class TestSimpleSignal { + public static AlgorithmConfiguration temperatureSignalConfig() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setAgent("TIME_SERIES_PROCESSING"); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + // vessels + config.setParam("TimeSeriesTable", "timeseries_id08b3abb9_c7b0_4b82_8117_64b69055416f"); + config.setParam("ValueColum", "fvalue"); + + + config.setParam("FrequencyResolution", "0.01"); + config.setParam("SSA_Window_in_Samples", "10"); + config.setParam("SSA_EigenvaluesThreshold", "0.7"); + config.setParam("SSA_Points_to_Forecast", "10"); + + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + config.setGcubeScope("/gcube"); + config.setConfigPath("./cfg"); + return config; + + } + } diff --git a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java index 47b6775..0aebf16 100644 --- a/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java +++ b/src/main/java/org/gcube/dataanalysis/ecoengine/transducers/TimeSeriesAnalysis.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import org.gcube.contentmanagement.graphtools.utils.DateGuesser; import org.gcube.contentmanagement.graphtools.utils.MathFunctions; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; @@ -48,7 +49,7 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { private Image forecastsignalImg = null; private Image eigenValuesImg=null; private File outputfilename=null; - private static boolean display = false; + public static boolean display = true; private static int maxpoints = 10000; @Override public void init() throws Exception { @@ -129,9 +130,17 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { TimeSeries ts = TimeSeries.buildFromSignal(signal, config); String timepattern = ts.getTimepattern(); String chartpattern = "HH:mm:ss MM-dd-yy"; - if (timepattern.equals("s")) + if (timepattern.equals("s") || + (DateGuesser.isJavaDateOrigin(ts.getTime()[0]) && + DateGuesser.isJavaDateOrigin(ts.getTime()[ts.getTime().length-1])) + ) + { + AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Changing chart pattern to Seconds!"); chartpattern = "HH:mm:ss:SS"; - + } + else + AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Chart pattern remains "+chartpattern); + AnalysisLogger.getLogger().debug("TimeSeriesAnalysis->Uniformly sampling the signal"); if (display) SignalProcessing.displaySignalWithTime(ts.getValues(), ts.getTime(), "Time Series", chartpattern); @@ -286,4 +295,7 @@ public class TimeSeriesAnalysis extends StandardLocalExternalAlgorithm { } + + + }