commit 8c8d1c31674328535dc92057729cd7c24a6b3f8a Author: Luca Frosini Date: Fri Feb 13 14:29:19 2015 +0000 Branching to have a new version depending on new SmartExecutor git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineSmartExecutor@112013 82a268e6-3cf1-43bd-a215-b396298e98cf diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..756245a --- /dev/null +++ b/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..3b4621b --- /dev/null +++ b/.project @@ -0,0 +1,23 @@ + + + EcologicalEngineExecutor + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.m2e.core.maven2Nature + org.eclipse.jdt.core.javanature + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..5c66f0e --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,4 @@ +#Fri Jun 22 18:05:41 CEST 2012 +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding/=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..6916b10 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,13 @@ +#Fri Jun 22 18:05:41 CEST 2012 +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..a09cbec --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,5 @@ +#Fri Jun 22 17:51:31 CEST 2012 +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/AQUAMAPS_SUITABLE/DestinationDBHibernate.cfg.xml b/AQUAMAPS_SUITABLE/DestinationDBHibernate.cfg.xml new file mode 100644 index 0000000..99feeae --- /dev/null +++ b/AQUAMAPS_SUITABLE/DestinationDBHibernate.cfg.xml @@ -0,0 +1,17 @@ + + + + org.postgresql.Driver + org.hibernate.connection.C3P0ConnectionProvider + jdbc:postgresql://localhost/testdb + gcube + d4science2 + org.hibernate.dialect.PostgreSQLDialect + org.hibernate.transaction.JDBCTransactionFactory + 0 + 1 + 0 + 1 + thread + + \ No newline at end of file diff --git a/AQUAMAPS_SUITABLE/aquamapsnode.jar b/AQUAMAPS_SUITABLE/aquamapsnode.jar new file mode 100644 index 0000000..68cde48 Binary files /dev/null and b/AQUAMAPS_SUITABLE/aquamapsnode.jar differ diff --git a/AQUAMAPS_SUITABLE/c3p0-0.9.1.2.jar b/AQUAMAPS_SUITABLE/c3p0-0.9.1.2.jar new file mode 100644 index 0000000..0f42d60 Binary files /dev/null and b/AQUAMAPS_SUITABLE/c3p0-0.9.1.2.jar differ diff --git a/AQUAMAPS_SUITABLE/commons-collections-3.1.jar b/AQUAMAPS_SUITABLE/commons-collections-3.1.jar new file mode 100644 index 0000000..41e230f Binary files /dev/null and b/AQUAMAPS_SUITABLE/commons-collections-3.1.jar differ diff --git a/AQUAMAPS_SUITABLE/dom4j-1.6.1.jar b/AQUAMAPS_SUITABLE/dom4j-1.6.1.jar new file mode 100644 index 0000000..c8c4dbb Binary files /dev/null and b/AQUAMAPS_SUITABLE/dom4j-1.6.1.jar differ diff --git a/AQUAMAPS_SUITABLE/ecologicalDataMining.jar b/AQUAMAPS_SUITABLE/ecologicalDataMining.jar new file mode 100644 index 0000000..b6527e7 Binary files /dev/null and b/AQUAMAPS_SUITABLE/ecologicalDataMining.jar differ diff --git a/AQUAMAPS_SUITABLE/hibernate3.jar b/AQUAMAPS_SUITABLE/hibernate3.jar new file mode 100644 index 0000000..f97dcdb Binary files /dev/null and b/AQUAMAPS_SUITABLE/hibernate3.jar differ diff --git a/AQUAMAPS_SUITABLE/jaxen-1.1.2.jar b/AQUAMAPS_SUITABLE/jaxen-1.1.2.jar new file mode 100644 index 0000000..69de309 Binary files /dev/null and b/AQUAMAPS_SUITABLE/jaxen-1.1.2.jar differ diff --git a/AQUAMAPS_SUITABLE/jta-1.1.jar b/AQUAMAPS_SUITABLE/jta-1.1.jar new file mode 100644 index 0000000..6d225b7 Binary files /dev/null and b/AQUAMAPS_SUITABLE/jta-1.1.jar differ diff --git a/AQUAMAPS_SUITABLE/log4j-1.2.16.jar b/AQUAMAPS_SUITABLE/log4j-1.2.16.jar new file mode 100644 index 0000000..3f9d847 Binary files /dev/null and b/AQUAMAPS_SUITABLE/log4j-1.2.16.jar differ diff --git a/AQUAMAPS_SUITABLE/postgresql-8.4-702.jdbc4.jar b/AQUAMAPS_SUITABLE/postgresql-8.4-702.jdbc4.jar new file mode 100644 index 0000000..8b0c761 Binary files /dev/null and b/AQUAMAPS_SUITABLE/postgresql-8.4-702.jdbc4.jar differ diff --git a/AQUAMAPS_SUITABLE/script.sh b/AQUAMAPS_SUITABLE/script.sh new file mode 100644 index 0000000..a797e76 --- /dev/null +++ b/AQUAMAPS_SUITABLE/script.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# AQUAMAPS_SUITABLE +cd $1 + +java -Xmx1024M -classpath ./:./aquamapsnode.jar:./c3p0-0.9.1.2.jar:./commons-collections-3.1.jar:./dom4j-1.6.1.jar:./ecologicalDataMining.jar:./hibernate3.jar:./jaxen-1.1.2.jar:./jta-1.1.jar:./log4j-1.2.16.jar:./postgresql-8.4-702.jdbc4.jar:./slf4j-api-1.6.0.jar:./slf4j-log4j12-1.6.0.jar:./xpp3_min-1.1.4c.jar:./xstream-1.3.1.jar org.gcube.dataanalysis.executor.nodes.algorithms.AquamapsSuitableNode $2 execution.output diff --git a/AQUAMAPS_SUITABLE/slf4j-api-1.6.0.jar b/AQUAMAPS_SUITABLE/slf4j-api-1.6.0.jar new file mode 100644 index 0000000..db92f9a Binary files /dev/null and b/AQUAMAPS_SUITABLE/slf4j-api-1.6.0.jar differ diff --git a/AQUAMAPS_SUITABLE/slf4j-log4j12-1.6.0.jar b/AQUAMAPS_SUITABLE/slf4j-log4j12-1.6.0.jar new file mode 100644 index 0000000..42cca51 Binary files /dev/null and b/AQUAMAPS_SUITABLE/slf4j-log4j12-1.6.0.jar differ diff --git a/AQUAMAPS_SUITABLE/xpp3_min-1.1.4c.jar b/AQUAMAPS_SUITABLE/xpp3_min-1.1.4c.jar new file mode 100644 index 0000000..813a9a8 Binary files /dev/null and b/AQUAMAPS_SUITABLE/xpp3_min-1.1.4c.jar differ diff --git a/AQUAMAPS_SUITABLE/xstream-1.3.1.jar b/AQUAMAPS_SUITABLE/xstream-1.3.1.jar new file mode 100644 index 0000000..4ef4219 Binary files /dev/null and b/AQUAMAPS_SUITABLE/xstream-1.3.1.jar differ diff --git a/PARALLEL_PROCESSING/CMSY_22_noplot.R b/PARALLEL_PROCESSING/CMSY_22_noplot.R new file mode 100644 index 0000000..6541ff6 --- /dev/null +++ b/PARALLEL_PROCESSING/CMSY_22_noplot.R @@ -0,0 +1,696 @@ +##-------------------------------------------------------- +## CMSY analysis with estimation of total biomass, including Bayesian Schaefer +## written by Rainer Froese with support from Gianpaolo Coro in 2013-2014 +## This version adjusts biomass to average biomass over the year +## It also contains the FutureCrash option to improve prediction of final biomass +## Version 21 adds the purple point to indicate the 25th percentile of final biomass +## Version 22 accepts that no biomass or CPUE area available +##-------------------------------------------------------- +library(R2jags) # Interface with JAGS +library(coda) + +#----------------------------------------- +# Some general settings +#----------------------------------------- +# set.seed(999) # use for comparing results between runs +rm(list=ls(all=TRUE)) # clear previous variables etc +options(digits=3) # displays all numbers with three significant digits as default +graphics.off() # close graphics windows from previous sessions + +#----------------------------------------- +# General settings for the analysis +#----------------------------------------- +sigR <- 0.02 # overall process error; 0.05 works reasonable for simulations, 0.02 for real data; 0 if deterministic model +n <- 10000 # initial number of r-k pairs +batch.mode <- T # set to TRUE to suppress graphs +write.output <- T # set to true if table of output is wanted +FutureCrash <- "No" + +#----------------------------------------- +# Start output to screen +#----------------------------------------- +cat("-------------------------------------------\n") +cat("Catch-MSY Analysis,", date(),"\n") +cat("-------------------------------------------\n") + +#------------------------------------------ +# Read data and assign to vectors +#------------------------------------------ +# filename_1 <- "AllStocks_Catch4.csv" +# filename_2 <- "AllStocks_ID4.csv" +# filename_1 <- "SimCatch.csv" +# filename_2 <- "SimSpec.csv" +# filename_2 <- "SimSpecWrongS.csv" +# filename_2 <- "SimSpecWrongI.csv" +# filename_2 <- "SimSpecWrongF.csv" +# filename_2 <- "SimSpecWrongH.csv" +# filename_2 <- "SimSpecWrongL.csv" +# filename_1 <- "FishDataLim.csv" +# filename_2 <- "FishDataLimSpec.csv" + filename_1 <- "WKLIFE4Stocks.csv" + filename_2 <- "WKLIFE4ID.csv" + +outfile<-"outfile" +outfile.txt <- "outputfile.txt" + +cdat <- read.csv(filename_1, header=T, dec=".", stringsAsFactors = FALSE) +cinfo <- read.csv(filename_2, header=T, dec=".", stringsAsFactors = FALSE) +cat("Files", filename_1, ",", filename_2, "read successfully","\n") + +# Stocks with total biomass data and catch data from StartYear to EndYear +# stocks <- sort(as.character(cinfo$stock)) # All stocks +stocks<-"HLH_M07" + +# select one stock after the other +for(stock in stocks) { + # assign data from cinfo to vectors + res <- as.character(cinfo$Resilience[cinfo$stock==stock]) + StartYear <- as.numeric(cinfo$StartYear[cinfo$stock==stock]) + EndYear <- as.numeric(cinfo$EndYear[cinfo$stock==stock]) + r_low <- as.numeric(cinfo$r_low[cinfo$stock==stock]) + r_hi <- as.numeric(cinfo$r_hi[cinfo$stock==stock]) + stb_low <- as.numeric(cinfo$stb_low[cinfo$stock==stock]) + stb_hi <- as.numeric(cinfo$stb_hi[cinfo$stock==stock]) + intyr <- as.numeric(cinfo$intyr[cinfo$stock==stock]) + intbio_low <- as.numeric(cinfo$intbio_low[cinfo$stock==stock]) + intbio_hi <- as.numeric(cinfo$intbio_hi[cinfo$stock==stock]) + endbio_low <- as.numeric(cinfo$endbio_low[cinfo$stock==stock]) + endbio_hi <- as.numeric(cinfo$endbio_hi[cinfo$stock==stock]) + Btype <- as.character(cinfo$Btype[cinfo$stock==stock]) + FutureCrash <- as.character(cinfo$FutureCrash[cinfo$stock==stock]) + comment <- as.character(cinfo$comment[cinfo$stock==stock]) + + + # extract data on stock + yr <- as.numeric(cdat$yr[cdat$stock==stock & cdat$yr >= StartYear & cdat$yr <= EndYear]) + ct <- as.numeric(cdat$ct[cdat$stock==stock & cdat$yr >= StartYear & cdat$yr <= EndYear])/1000 ## assumes that catch is given in tonnes, transforms to '000 tonnes + if(Btype=="observed" | Btype=="CPUE" | Btype=="simulated") { + bt <- as.numeric(cdat$TB[cdat$stock==stock & cdat$yr >= StartYear & cdat$yr <= EndYear])/1000 ## assumes that biomass is in tonnes, transforms to '000 tonnes + } else {bt <- NA} + nyr <- length(yr) # number of years in the time series + + +if(Btype!="observed") {bio <- bt} +# change biomass to moving average as assumed by Schaefer (but not for simulations or CPUE) +# for last year use reported bio +if(Btype=="observed") { + ma <- function(x){filter(x,rep(1/2,2),sides=2)} + bio <- ma(bt) + bio[length(bio)] <- bt[length(bt)] } + + # initialize vectors for viable r, k, bt + rv.all <- vector() + kv.all <- vector() + btv.all <- matrix(data=vector(),ncol=nyr+1) + + + + #---------------------------------------------------- + # Determine initial ranges for parameters and biomass + #---------------------------------------------------- + # initial range of r from input file + if(is.na(r_low)==F & is.na(r_hi)==F) { + start_r <- c(r_low,r_hi) + } else { + # initial range of r and CatchMult values based on resilience + if(res == "High") { + start_r <- c(0.6,1.5)} else if(res == "Medium") { + start_r <- c(0.2,0.8)} else if(res == "Low") { + start_r <- c(0.05,0.5)} else { # i.e. res== "Very low" + start_r <- c(0.015,0.1)} + } + + + # initial range of k values, assuming k will always be larger than max catch + # and max catch will never be smaller than a quarter of MSY + + start_k <- c(max(ct),16*max(ct)/start_r[1]) + + # initial biomass range from input file + if(is.na(stb_low)==F & is.na(stb_hi)==F) { + startbio <- c(stb_low,stb_hi) + } else { + # us low biomass at start as default + startbio <- c(0.1,0.5) + } + + MinYear <- yr[which.min(ct)] + MaxYear <- yr[which.max(ct)] + # use year and biomass range for intermediate biomass from input file + if(is.na(intbio_low)==F & is.na(intbio_hi)==F) { + intyr <- intyr + intbio <- c(intbio_low,intbio_hi) + # else if year of minimum catch is at least 3 years away from StartYear and EndYear of series, use min catch + } else if((MinYear - StartYear) > 3 & (EndYear - MinYear) > 3 ) { + # assume that biomass range in year before minimum catch was 0.01 - 0.4 + intyr <- MinYear-1 + intbio <- c(0.01,0.4) + # else if year of max catch is at least 3 years away from StartYear and EndYear of series, use max catch + } else if((MaxYear - StartYear) > 3 & (EndYear - MaxYear) > 3 ) { + # assume that biomass range in year before maximum catch was 0.3 - 0.9 + intyr <- MaxYear-1 + intbio <- c(0.3,0.9) + } else { + # assume uninformative range 0-1 in mid-year + intyr <- as.integer(mean(c(StartYear, EndYear))) + intbio <- c(0,1) } + # end of intbio setting + + # final biomass range from input file + if(is.na(endbio_low)==F & is.na(endbio_hi)==F) { + endbio <- c(endbio_low,endbio_hi) + } else { + # else use Catch/maxCatch to estimate final biomass + endbio <- if(ct[nyr]/max(ct) > 0.5) {c(0.4,0.8)} else {c(0.01,0.4)} + } # end of final biomass setting + + + #---------------------------------------------- + # MC with Schaefer Function filtering + #---------------------------------------------- + Schaefer <- function(ri, ki, startbio, intyr, intbio, endbio, sigR, pt) { + + # if stock is not expected to crash within 3 years if last catch continues + if(FutureCrash == "No") { + yr.s <- c(yr,EndYear+1,EndYear+2,EndYear+3) + ct.s <- c(ct,ct[yr==EndYear],ct[yr==EndYear],ct[yr==EndYear]) + nyr.s <- length(yr.s) + } else{ + yr.s <- yr + ct.s <- ct + nyr.s <- nyr + } + + # create vector for initial biomasses + startbt <-seq(from =startbio[1], to=startbio[2], by = (startbio[2]-startbio[1])/10) + # create vectors for viable r, k and bt + rv <- array(-1:-1,dim=c(length(ri)*length(startbt))) #initialize array with -1. The -1 remaining after the process will be removed + kv <- array(-1:-1,dim=c(length(ri)*length(startbt))) + btv <- matrix(data=NA, nrow = (length(ri)*length(startbt)), ncol = nyr+1) + intyr.i <- which(yr.s==intyr) # get index of intermediate year + + #loop through r-k pairs + npoints = length(ri) + nstartb = length(startbt) + + for(i in 1 : npoints) { + if (i%%1000==0) + cat(".") + + # create empty vector for annual biomasses + bt <- vector() + + # loop through range of relative start biomasses + for(j in startbt) { + # set initial biomass, including process error + bt[1]=j*ki[i]*exp(rnorm(1,0, sigR)) ## set biomass in first year + + #loop through years in catch time series + for(t in 1:nyr.s) { # for all years in the time series + xt=rnorm(1,0, sigR) # set new random process error for every year + + # calculate biomass as function of previous year's biomass plus surplus production minus catch + bt[t+1]=(bt[t]+ri[i]*bt[t]*(1-bt[t]/ki[i])-ct.s[t])*exp(xt) + + # if biomass < 0.01 k or > 1.1 k, discard r-k pair + if(bt[t+1] < 0.01*ki[i] || bt[t+1] > 1.1*ki[i]) { break } # stop looping through years, go to next upper level + + if ((t+1)==intyr.i && (bt[t+1]>(intbio[2]*ki[i]) || bt[t+1]<(intbio[1]*ki[i]))) { break } #intermediate year check + + } # end of loop of years + + # if last biomass falls without expected ranges goto next r-k pair + if(t < nyr.s || bt[yr.s==EndYear] > (endbio[2]*ki[i]) || bt[yr.s==EndYear] < (endbio[1]*ki[i])) { + next } else { + # store r, k, and bt, plot point, then go to next startbt + rv[((i-1)*nstartb)+j] <- ri[i] + kv[((i-1)*nstartb)+j] <- ki[i] + btv[((i-1)*nstartb)+j,] <- bt[1:(nyr+1)]/ki[i] #substitute a row into the matrix, exclude FutureCrash years + if(pt==T) {points(x=ri[i], y=ki[i], pch=".", cex=2, col="black") + next } + } + } # end of loop of initial biomasses + } # end of loop of r-k pairs + + rv=rv[rv!=-1] + kv=kv[kv!=-1] + btv=na.omit(btv) #delete first line + + cat("\n") + return(list(rv, kv,btv)) + } # end of Schaefer function + + #------------------------------------------------------------------ + # Uniform sampling of the r-k space + #------------------------------------------------------------------ + # get random set of r and k from log space distribution + ri1 = exp(runif(n, log(start_r[1]), log(start_r[2]))) + ki1 = exp(runif(n, log(start_k[1]), log(start_k[2]))) + + #----------------------------------------------------------------- + # Plot data and progress + #----------------------------------------------------------------- + #windows(14,9) + par(mfcol=c(2,3)) + # plot catch + plot(x=yr, y=ct, ylim=c(0,1.2*max(ct)), type ="l", bty="l", main=paste(stock,"catch"), xlab="Year", + ylab="Catch", lwd=2) + points(x=yr[which.max(ct)], y=max(ct), col="red", lwd=2) + points(x=yr[which.min(ct)], y=min(ct), col="red", lwd=2) + + # plot r-k graph + plot(ri1, ki1, xlim = start_r, ylim = start_k, log="xy", xlab="r", ylab="k", main="Finding viable r-k", pch=".", cex=2, bty="l", col="lightgrey") + + #1 - Call MC-Schaefer function to preliminary explore the space without prior information + cat(stock, ": First Monte Carlo filtering of r-k space with ",n," points\n") + MCA <- Schaefer(ri=ri1, ki=ki1, startbio=startbio, intyr=intyr, intbio=intbio, endbio=endbio, sigR=sigR, pt=T) + rv.all <- append(rv.all,MCA[[1]]) + kv.all <- append(kv.all,MCA[[2]]) + btv.all <- rbind(btv.all,MCA[[3]]) + #take viable r and k values + nviablepoints = length(rv.all) + cat("* Found ",nviablepoints," viable points from ",n," samples\n"); + + + #if few points were found then resample and shrink the k log space + if (nviablepoints<=1000){ + log.start_k.new <- log(start_k) + max_attempts = 3 + current_attempts = 1 + while (nviablepoints<=1000 && current_attempts<=max_attempts){ + if(nviablepoints > 0) { + log.start_k.new[1] <- mean(c(log.start_k.new[1], min(log(kv.all)))) + log.start_k.new[2] <- mean(c(log.start_k.new[2], max(log(kv.all)))) } + n.new=n*current_attempts #add more points + ri1 = exp(runif(n.new, log(start_r[1]), log(start_r[2]))) + ki1 = exp(runif(n.new, log.start_k.new[1], log.start_k.new[2])) + cat("Shrinking k space: repeating Monte Carlo in the interval [",exp(log.start_k.new[1]),",",exp(log.start_k.new[2]),"]\n") + cat("Attempt ",current_attempts," of ",max_attempts," with ",n.new," points","\n") + MCA <- Schaefer(ri=ri1, ki=ki1, startbio=startbio, intyr=intyr, intbio=intbio, endbio=endbio, sigR=sigR, pt=T) + rv.all <- append(rv.all,MCA[[1]]) + kv.all <- append(kv.all,MCA[[2]]) + btv.all <- rbind(btv.all,MCA[[3]]) + nviablepoints = length(rv.all) #recalculate viable points + cat("* Found altogether",nviablepoints," viable points \n"); + current_attempts=current_attempts+1 #increment the number of attempts + } + } + + # If tip of viable r-k pairs is 'thin', do extra sampling there + gm.rv = exp(mean(log(rv.all))) + if(length(rv.all[rv.all > 0.9*start_r[2]]) < 10) { + l.sample.r <- (gm.rv + max(rv.all))/2 + cat("Final sampling in the tip area above r =",l.sample.r,"\n") + log.start_k.new <- c(log(0.8*min(kv.all)),log(max(kv.all[rv.all > l.sample.r]))) + ri1 = exp(runif(50000, log(l.sample.r), log(start_r[2]))) + ki1 = exp(runif(50000, log.start_k.new[1], log.start_k.new[2])) + MCA <- Schaefer(ri=ri1, ki=ki1, startbio=startbio, intyr=intyr, intbio=intbio, endbio=endbio, sigR=sigR, pt=T) + rv.all <- append(rv.all,MCA[[1]]) + kv.all <- append(kv.all,MCA[[2]]) + btv.all <- rbind(btv.all,MCA[[3]]) + nviablepoints = length(rv.all) #recalculate viable points + cat("Found altogether", length(rv.all), "unique viable r-k pairs and biomass trajectories\n") + } + + + # ------------------------------------------------------------ + # Bayesian analysis of catch & biomass with Schaefer model + # ------------------------------------------------------------ + if(Btype == "observed" | Btype=="simulated") { + cat("Running Schaefer MCMC analysis....\n") + mcmc.burn <- as.integer(30000) + mcmc.chainLength <- as.integer(60000) # burn-in plus post-burn + mcmc.thin = 10 # to reduce autocorrelation + mcmc.chains = 3 # needs to be at least 2 for DIC + + # Parameters to be returned by JAGS + jags.save.params=c('r','k','sigma.b', 'alpha', 'sigma.r') # + + # JAGS model + Model = "model{ + # to avoid crash due to 0 values + eps<-0.01 + # set a quite narrow variation from the expected value + sigma.b <- 1/16 + tau.b <- pow(sigma.b,-2) + + Bm[1] <- log(alpha*k) + bio[1] ~ dlnorm(Bm[1],tau.b) + + + for (t in 2:nyr){ + bio[t] ~ dlnorm(Bm[t],tau.b) + Bm[t] <- log(max(bio[t-1] + r*bio[t-1]*(1 - (bio[t-1])/k) - ct[t-1], eps)) + } + + # priors + alpha ~ dunif(0.01,1) # needed for fit of first biomass + #inverse cubic root relationship between the range of viable r and the size of the search space + inverseRangeFactor <- 1/((start_r[2]-start_r[1])^1/3) + + # give sigma some variability in the inverse relationship + sigma.r ~ dunif(0.001*inverseRangeFactor,0.02*inverseRangeFactor) + tau.r <- pow(sigma.r,-2) + rm <- log((start_r[1]+start_r[2])/2) + r ~ dlnorm(rm,tau.r) + + # search in the k space from the center of the range. Allow high variability + km <- log((start_k[1]+start_k[2])/2) + tau.k <- pow(km,-2) + k ~ dlnorm(km,tau.k) + + #end model + }" + + # Write JAGS model to file + cat(Model, file="r2jags.bug") + + ### random seed + set.seed(runif(1,1,500)) # needed in JAGS + + ### run model + jags_outputs <- jags(data=c('ct','bio','nyr', 'start_r', 'start_k'), + working.directory=NULL, inits=NULL, + parameters.to.save= jags.save.params, + model.file="r2jags.bug", n.chains = mcmc.chains, + n.burnin = mcmc.burn, n.thin = mcmc.thin, n.iter = mcmc.chainLength, + refresh=mcmc.burn/20, ) + + # ------------------------------------------------------ + # Results from JAGS Schaefer + # ------------------------------------------------------ + r_out <- as.numeric(mcmc(jags_outputs$BUGSoutput$sims.list$r)) + k_out <- as.numeric(mcmc(jags_outputs$BUGSoutput$sims.list$k)) +## sigma_out <- as.numeric(mcmc(jags_outputs$BUGSoutput$sims.list$sigma.b)) + alpha_out <- as.numeric(mcmc(jags_outputs$BUGSoutput$sims.list$alpha)) +## sigma.r_out <- as.numeric(mcmc(jags_outputs$BUGSoutput$sims.list$sigma.r)) + + mean.log.r.jags <- mean(log(r_out)) + SD.log.r.jags <- sd(log(r_out)) + lcl.log.r.jags <- mean.log.r.jags-1.96*SD.log.r.jags + ucl.log.r.jags <- mean.log.r.jags+1.96*SD.log.r.jags + gm.r.jags <- exp(mean.log.r.jags) + lcl.r.jags <- exp(lcl.log.r.jags) + ucl.r.jags <- exp(ucl.log.r.jags) + mean.log.k.jags <- mean(log(k_out)) + SD.log.k.jags <- sd(log(k_out)) + lcl.log.k.jags <- mean.log.k.jags-1.96*SD.log.k.jags + ucl.log.k.jags <- mean.log.k.jags+1.96*SD.log.k.jags + gm.k.jags <- exp(mean.log.k.jags) + lcl.k.jags <- exp(lcl.log.k.jags) + ucl.k.jags <- exp(ucl.log.k.jags) + mean.log.MSY.jags<- mean(log(r_out)+log(k_out)-log(4)) + SD.log.MSY.jags <- sd(log(r_out)+log(k_out)-log(4)) + gm.MSY.jags <- exp(mean.log.MSY.jags) + lcl.MSY.jags <- exp(mean.log.MSY.jags-1.96*SD.log.MSY.jags) + ucl.MSY.jags <- exp(mean.log.MSY.jags+1.96*SD.log.MSY.jags) + +} # end of MCMC Schaefer loop + +#------------------------------------ +# get results from CMSY +#------------------------------------ +# get estimate of most probable r as median of mid log.r-classes above cut-off +# get remaining viable log.r and log.k +rem.log.r <- log(rv.all[rv.all > gm.rv]) +rem.log.k <- log(kv.all[rv.all>gm.rv]) +# get vectors with numbers of r and mid values in about 25 classes +hist.log.r <- hist(x=rem.log.r, breaks=25, plot=F) +log.r.counts <- hist.log.r$counts +log.r.mids <- hist.log.r$mids +# get most probable log.r as mean of mids with counts > 0 +log.r.est <- median(log.r.mids[which(log.r.counts > 0)]) +lcl.log.r <- as.numeric(quantile(x=log.r.mids[which(log.r.counts > 0)], 0.025)) +ucl.log.r <- as.numeric(quantile(x=log.r.mids[which(log.r.counts > 0)], 0.975)) +r.est <- exp(log.r.est) +lcl.r.est <- exp(lcl.log.r) +ucl.r.est <- exp(ucl.log.r) + +# do linear regression of log k ~ log r with slope fixed to -1 (from Schaefer) +reg <- lm(rem.log.k ~ 1 + offset(-1*rem.log.r)) +int.reg <- as.numeric(reg[1]) +sd.reg <- sd(resid(reg)) +se.reg <- summary(reg)$coefficients[2] +# get estimate of log(k) from y where x = log.r.est +log.k.est <- int.reg + (-1) * log.r.est +# get estimates of CL of log.k.est from y +/- SD where x = lcl.log r or ucl.log.r +lcl.log.k <- int.reg + (-1) * ucl.log.r - sd.reg +ucl.log.k <- int.reg + (-1) * lcl.log.r + sd.reg +k.est <- exp(log.k.est) +lcl.k.est <- exp(lcl.log.k) +ucl.k.est <- exp(ucl.log.k) + +# get MSY from remaining log r-k pairs +log.MSY.est <- mean(rem.log.r + rem.log.k - log(4)) +sd.log.MSY.est <- sd(rem.log.r + rem.log.k - log(4)) +lcl.log.MSY.est <- log.MSY.est - 1.96*sd.log.MSY.est +ucl.log.MSY.est <- log.MSY.est + 1.96*sd.log.MSY.est +MSY.est <- exp(log.MSY.est) +lcl.MSY.est <- exp(lcl.log.MSY.est) +ucl.MSY.est <- exp(ucl.log.MSY.est) + +# get predicted biomass vectors as median and quantiles of trajectories +median.btv <- apply(btv.all,2, median) +lastyr.bio <- median.btv[length(median.btv)-1] +nextyr.bio <- median.btv[length(median.btv)] +lcl.btv <- apply(btv.all,2, quantile, probs=0.025) +q.btv <- apply(btv.all,2, quantile, probs=0.25) +ucl.btv <- apply(btv.all,2, quantile, probs=0.975) +lcl.lastyr.bio <- lcl.btv[length(lcl.btv)-1] +ucl.lastyr.bio <- ucl.btv[length(lcl.btv)-1] +lcl.nextyr.bio <- lcl.btv[length(lcl.btv)] +ucl.nextyr.bio <- ucl.btv[length(lcl.btv)] + +# ----------------------------------------- +# Plot results +# ----------------------------------------- +# Analysis of viable r-k pairs +plot(x=rv.all, y=kv.all, xlim=start_r, + ylim=c(0.9*min(kv.all, ifelse(Btype == "observed",k_out,NA), na.rm=T), 1.1*max(kv.all)), + pch=16, col="grey",log="xy", bty="l", + xlab="r", ylab="k", main="Analysis of viable r-k") +abline(v=gm.rv, lty="dashed") + +# plot points and best estimate from full Schaefer analysis +if(Btype == "observed"|Btype=="simulated") { + # plot r-k pairs from MCMC + points(x=r_out, y=k_out, pch=16,cex=0.5) + # plot best r-k pair from MCMC + points(x=gm.r.jags, y=gm.k.jags, pch=19, col="green") + lines(x=c(lcl.r.jags, ucl.r.jags),y=c(gm.k.jags,gm.k.jags), col="green") + lines(x=c(gm.r.jags,gm.r.jags),y=c(lcl.k.jags, ucl.k.jags), col="green") +} + +# if data are from simulation, plot true r and k +if(Btype=="simulated") { + l.stock <- nchar(stock) # get length of sim stock name + r.char <- substr(stock,l.stock-1,l.stock) # get last character of sim stock name + r.sim <- NA # initialize vector for r used in simulation + if(r.char=="_H") {r.sim=1; lcl.r.sim=0.8; ucl.r.sim=1.25} else + if(r.char=="_M") {r.sim=0.5;lcl.r.sim=0.4;ucl.r.sim=0.62} else + if(r.char=="_L") {r.sim=0.25;lcl.r.sim=0.2;ucl.r.sim=0.31} else {r.sim=0.05;lcl.r.sim=0.04;ucl.r.sim=0.062} + # plot true r-k point with error bars + points(x=r.sim, y=1000, pch=19, col="red") + # add +/- 20% error bars + lines(x=c(lcl.r.sim,ucl.r.sim), y=c(1000,1000), col="red") + lines(x=c(r.sim,r.sim), y=c(800,1250), col="red") +} + +# plot blue dot for proposed r-k, with 95% CL lines +points(x=r.est, y=k.est, pch=19, col="blue") +lines(x=c(lcl.r.est, ucl.r.est),y=c(k.est,k.est), col="blue") +lines(x=c(r.est,r.est),y=c(lcl.k.est, ucl.k.est), col="blue") + +# plot biomass graph +# determine k to use for red line in b/k plot +if(Btype=="simulated") {k2use <- 1000} else + if(Btype == "observed") {k2use <- gm.k.jags} else {k2use <- k.est} +# determine hight of y-axis in plot +max.y <- max(c(bio/k2use,ucl.btv,0.6,startbio[2], intbio[2],endbio[2]),na.rm=T) + +plot(x=yr,y=median.btv[1:nyr], lwd=2, xlab="Year", ylab="Relative biomass b/k", type="l", + ylim=c(0,max.y), bty="l", main=paste("Pred. biomass vs ", Btype,sep="")) +lines(x=yr, y=lcl.btv[1:nyr],type="l") +lines(x=yr, y=ucl.btv[1:nyr],type="l") +points(x=EndYear,y=q.btv[yr==EndYear], col="purple", cex=1.5, lwd=2) +abline(h=0.5, lty="dashed") +abline(h=0.25, lty="dotted") +lines(x=c(yr[1],yr[1]), y=startbio, col="blue") +lines(x=c(intyr,intyr), y=intbio, col="blue") +lines(x=c(max(yr),max(yr)), y=endbio, col="blue") + +# if observed biomass is available, plot red biomass line +if(Btype == "observed"|Btype=="simulated") { + lines(x=yr, y=bio/k2use,type="l", col="red", lwd=1) + } + +# if CPUE data are available, scale to predicted biomass range, plot red biomass line +if(Btype == "CPUE") { + par(new=T) # prepares for new plot on top of previous + plot(x=yr, y=bio, type="l", col="red", lwd=1, + ann=F,axes=F,ylim=c(0,1.2*max(bio, na.rm=T))) # forces this plot on top of previous one + axis(4, col="red", col.axis="red") +} + +# plot yield and biomass against equilibrium surplus parabola +max.y <-max(c(ct/MSY.est,ifelse(Btype=="observed"|Btype=="simulated",ct/gm.MSY.jags,NA),1.2),na.rm=T) +# plot parabola +x=seq(from=0,to=2,by=0.001) +y=4*x-(2*x)^2 +plot(x=x, y=y, xlim=c(0,1), ylim=c(0,max.y), type="l", bty="l",xlab="Relative biomass b/k", + ylab="Catch / MSY", main="Equilibrium curve") +# plot catch against CMSY biomass estimates +points(x=median.btv[1:nyr], y=ct/MSY.est, pch=16, col="grey") +points(x=q.btv[yr==EndYear],y=ct[yr==EndYear]/MSY.est, col="purple", cex=1.5, lwd=2) +# plot catch against observed biomass or CPUE +if(Btype == "observed"|Btype=="simulated") { + points(x=bio/k2use, y=ct/gm.MSY.jags, pch=16, cex=0.5) +} + +# plot exploitation rate u against u.msy +# get u derived from predicted CMSY biomass +u.CMSY <- ct/(median.btv[1:nyr]*k.est) +u.msy.CMSY <- 1-exp(-r.est/2) # # Fmsy from CMSY expressed as exploitation rate +# get u from observed or simulated biomass +if(Btype == "observed"|Btype=="simulated") { + u.bio <- ct/bio + u.msy.bio <- 1-exp(-gm.r.jags/2) +} +# get u from CPUE +if(Btype == "CPUE") { + q=max(median.btv[1:nyr][is.na(bio)==F],na.rm=T)*k.est/max(bio,na.rm=T) + u.CPUE <- ct/(q*bio) +} + +# determine upper bound of Y-axis +max.y <- max(c(1.5, 1.2*u.CMSY/u.msy.CMSY,ct[yr==EndYear]/(q.btv[yr==EndYear]*k.est)/u.msy.CMSY, + ifelse(Btype=="observed"|Btype=="simulated",max(u.bio[is.na(u.bio)==F]/u.msy.bio),0), + na.rm=T)) +# plot u from CMSY +plot(x=yr,y=u.CMSY/u.msy.CMSY, type="l", bty="l", ylim=c(0,max.y), xlab="Year", + ylab="u / u_msy", main="Exploitation rate") +abline(h=1, lty="dashed") +points(x=EndYear,y=ct[yr==EndYear]/(q.btv[yr==EndYear]*k.est)/u.msy.CMSY, col="purple", cex=1.5, lwd=2) +# plot u from biomass +if(Btype == "observed"|Btype=="simulated") lines(x=yr, y=u.bio/u.msy.bio, col="red") +# plot u from CPUE +if(Btype == "CPUE") { + par(new=T) # prepares for new plot on top of previous + plot(x=yr, y=u.CPUE, type="l", col="red", ylim=c(0, 1.2*max(u.CPUE,na.rm=T)),ann=F,axes=F) + axis(4, col="red", col.axis="red") +} +if(batch.mode == TRUE) {dev.off()} # close plot window + +# ------------------------------------------ +# print input and results to screen +cat("---------------------------------------\n") + +cat("Species:", cinfo$ScientificName[cinfo$stock==stock], "\n") +cat("Name and region:", cinfo$EnglishName[cinfo$stock==stock], ",", cinfo$Name[cinfo$stock==stock], "\n") +cat("Stock:",stock,"\n") +cat("Catch data used from years", min(yr),"-", max(yr), "\n") +cat("Prior initial relative biomass =", startbio[1], "-", startbio[2], "\n") +cat("Prior intermediate rel. biomass=", intbio[1], "-", intbio[2], "in year", intyr, "\n") +cat("Prior final relative biomass =", endbio[1], "-", endbio[2], "\n") +cat("If current catches continue, is the stock likely to crash within 3 years?",FutureCrash,"\n") +cat("Prior range for r =", format(start_r[1],digits=2), "-", format(start_r[2],digits=2), + ", prior range for k =", start_k[1], "-", start_k[2],"\n") + +# if data are simulated, print true r-k +if(filename_1=="SimCatch.csv") { +cat("True r =", r.sim, "(because input data were simulated with Schaefer model)\n") +cat("True k = 1000 \n") +cat("True MSY =", 1000*r.sim/4,"\n") +cat("True biomass in last year =",bio[length(bio)],"or",bio[length(bio)]/1000,"k \n") +cat("True mean catch / MSY ratio =", mean(ct)/(1000*r.sim/4),"\n") +} +# print results from full Schaefer if available +if(Btype == "observed"|Btype=="simulated") { +cat("Results from Bayesian Schaefer model using catch & biomass (",Btype,")\n") +cat("MSY =", gm.MSY.jags,", 95% CL =", lcl.MSY.jags, "-", ucl.MSY.jags,"\n") +cat("Mean catch / MSY =", mean(ct)/gm.MSY.jags,"\n") +if(Btype != "CPUE") { + cat("r =", gm.r.jags,", 95% CL =", lcl.r.jags, "-", ucl.r.jags,"\n") + cat("k =", gm.k.jags,", 95% CL =", lcl.k.jags, "-", ucl.k.jags,"\n") + } +} +# results of CMSY analysis +cat("Results of CMSY analysis \n") +cat("Altogether", nviablepoints,"unique viable r-k pairs were found \n") +cat(nviablepoints-length(rem.log.r),"r-k pairs above the initial geometric mean of r =", gm.rv, "were analysed\n") +cat("r =", r.est,", 95% CL =", lcl.r.est, "-", ucl.r.est,"\n") +cat("k =", k.est,", 95% CL =", lcl.k.est, "-", ucl.k.est,"\n") +cat("MSY =", MSY.est,", 95% CL =", lcl.MSY.est, "-", ucl.MSY.est,"\n") +cat("Predicted biomass in last year =", lastyr.bio, "2.5th perc =", lcl.lastyr.bio, + "97.5th perc =", ucl.lastyr.bio,"\n") +cat("Predicted biomass in next year =", nextyr.bio, "2.5th perc =", lcl.nextyr.bio, + "97.5th perc =", ucl.nextyr.bio,"\n") +cat("----------------------------------------------------------\n") + +## Write some results into outfile +if(write.output == TRUE) { +# write data into csv file + output = data.frame(cinfo$ScientificName[cinfo$stock==stock], stock, StartYear, EndYear, mean(ct)*1000, + ifelse(Btype=="observed"|Btype=="simulate",bio[length(bio)],NA), # last biomass on record + ifelse(Btype == "observed"|Btype=="simulated",gm.MSY.jags,NA), # full Schaefer + ifelse(Btype == "observed"|Btype=="simulated",lcl.MSY.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",ucl.MSY.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",gm.r.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",lcl.r.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",ucl.r.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",gm.k.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",lcl.k.jags,NA), + ifelse(Btype == "observed"|Btype=="simulated",ucl.k.jags,NA), + r.est, lcl.r.est, ucl.r.est, # CMSY r + k.est, lcl.k.est, ucl.k.est, # CMSY k + MSY.est, lcl.MSY.est, ucl.MSY.est, # CMSY r + lastyr.bio, lcl.lastyr.bio, ucl.lastyr.bio, # last year bio + nextyr.bio, lcl.nextyr.bio, ucl.nextyr.bio)# last year + 1 bio + + write.table(output, file=outfile, append = T, sep = ",", + dec = ".", row.names = FALSE, col.names = FALSE) + +# write some text into text outfile.txt + +cat("Species:", cinfo$ScientificName[cinfo$stock==stock], "\n", + "Name:", cinfo$EnglishName[cinfo$stock==stock], "\n", + "Region:", cinfo$Name[cinfo$stock==stock], "\n", + "Stock:",stock,"\n", + "Catch data used from years", min(yr),"-", max(yr),", biomass =", Btype, "\n", + "Prior initial relative biomass =", startbio[1], "-", startbio[2], "\n", + "Prior intermediate rel. biomass=", intbio[1], "-", intbio[2], "in year", intyr, "\n", + "Prior final relative biomass =", endbio[1], "-", endbio[2], "\n", + "Future crash with current catches?", FutureCrash, "\n", + "Prior range for r =", format(start_r[1],digits=2), "-", format(start_r[2],digits=2), + ", prior range for k =", start_k[1], "-", start_k[2],"\n", + file=outfile.txt,append=T) + + if(filename_1=="SimCatch.csv") { + cat(" True r =", r.sim, "(because input data were simulated with Schaefer model)\n", + "True k = 1000, true MSY =", 1000*r.sim/4,"\n", + "True biomass in last year =",bio[length(bio)],"or",bio[length(bio)]/1000,"k \n", + "True mean catch / MSY ratio =", mean(ct)/(1000*r.sim/4),"\n", + file=outfile.txt,append=T) + } + if(Btype == "observed"|Btype=="simulated") { + cat(" Results from Bayesian Schaefer model using catch & biomass \n", + "r =", gm.r.jags,", 95% CL =", lcl.r.jags, "-", ucl.r.jags,"\n", + "k =", gm.k.jags,", 95% CL =", lcl.k.jags, "-", ucl.k.jags,"\n", + "MSY =", gm.MSY.jags,", 95% CL =", lcl.MSY.jags, "-", ucl.MSY.jags,"\n", + "Mean catch / MSY =", mean(ct)/gm.MSY.jags,"\n", + file=outfile.txt,append=T) + } + cat(" Results of CMSY analysis with altogether", nviablepoints,"unique viable r-k pairs \n", + nviablepoints-length(rem.log.r),"r-k pairs above the initial geometric mean of r =", gm.rv, "were analysed\n", + "r =", r.est,", 95% CL =", lcl.r.est, "-", ucl.r.est,"\n", + "k =", k.est,", 95% CL =", lcl.k.est, "-", ucl.k.est,"\n", + "MSY =", MSY.est,", 95% CL =", lcl.MSY.est, "-", ucl.MSY.est,"\n", + "Predicted biomass last year b/k =", lastyr.bio, "2.5th perc b/k =", lcl.lastyr.bio, + "97.5th perc b/k =", ucl.lastyr.bio,"\n", + "Precautionary 25th percentile b/k =",q.btv[yr==EndYear],"\n", + "----------------------------------------------------------\n", + file=outfile.txt,append=T) + + } + +} # end of stocks loop diff --git a/PARALLEL_PROCESSING/CMSY_22_noplot.R.test b/PARALLEL_PROCESSING/CMSY_22_noplot.R.test new file mode 100644 index 0000000..b77ea9e --- /dev/null +++ b/PARALLEL_PROCESSING/CMSY_22_noplot.R.test @@ -0,0 +1,119 @@ +##-------------------------------------------------------- +## CMSY analysis with estimation of total biomass, including Bayesian Schaefer +## written by Rainer Froese with support from Gianpaolo Coro in 2013-2014 +## This version adjusts biomass to average biomass over the year +## It also contains the FutureCrash option to improve prediction of final biomass +## Version 21 adds the purple point to indicate the 25th percentile of final biomass +## Version 22 accepts that no biomass or CPUE area available +##-------------------------------------------------------- +library(R2jags) # Interface with JAGS +library(coda) + +#----------------------------------------- +# Some general settings +#----------------------------------------- +# set.seed(999) # use for comparing results between runs +rm(list=ls(all=TRUE)) # clear previous variables etc +options(digits=3) # displays all numbers with three significant digits as default +graphics.off() # close graphics windows from previous sessions + +#----------------------------------------- +# General settings for the analysis +#----------------------------------------- +sigR <- 0.02 # overall process error; 0.05 works reasonable for simulations, 0.02 for real data; 0 if deterministic model +n <- 10000 # initial number of r-k pairs +batch.mode <- T # set to TRUE to suppress graphs +write.output <- T # set to true if table of output is wanted +FutureCrash <- "No" + +#----------------------------------------- +# Start output to screen +#----------------------------------------- +cat("-------------------------------------------\n") +cat("Catch-MSY Analysis,", date(),"\n") +cat("-------------------------------------------\n") + +#------------------------------------------ +# Read data and assign to vectors +#------------------------------------------ +# filename_1 <- "AllStocks_Catch4.csv" +# filename_2 <- "AllStocks_ID4.csv" +# filename_1 <- "SimCatch.csv" +# filename_2 <- "SimSpec.csv" +# filename_2 <- "SimSpecWrongS.csv" +# filename_2 <- "SimSpecWrongI.csv" +# filename_2 <- "SimSpecWrongF.csv" +# filename_2 <- "SimSpecWrongH.csv" +# filename_2 <- "SimSpecWrongL.csv" +# filename_1 <- "FishDataLim.csv" +# filename_2 <- "FishDataLimSpec.csv" +filename_1 <- "WKLIFE4Stocks.csv" +filename_2 <- "WKLIFE4ID.csv" + +outfile<-"outfile" +outfile.txt <- "outputfile.txt" + +cdat <- read.csv(filename_1, header=T, dec=".", stringsAsFactors = FALSE) +cinfo <- read.csv(filename_2, header=T, dec=".", stringsAsFactors = FALSE) +cat("Files", filename_1, ",", filename_2, "read successfully","\n") + +# Stocks with total biomass data and catch data from StartYear to EndYear +# stocks <- sort(as.character(cinfo$stock)) # All stocks +stocks<-"HLH_M07" + +# select one stock after the other +for(stock in stocks) { + # assign data from cinfo to vectors + res <- as.character(cinfo$Resilience[cinfo$stock==stock]) + StartYear <- as.numeric(cinfo$StartYear[cinfo$stock==stock]) + EndYear <- as.numeric(cinfo$EndYear[cinfo$stock==stock]) + r_low <- as.numeric(cinfo$r_low[cinfo$stock==stock]) + r_hi <- as.numeric(cinfo$r_hi[cinfo$stock==stock]) + stb_low <- as.numeric(cinfo$stb_low[cinfo$stock==stock]) + stb_hi <- as.numeric(cinfo$stb_hi[cinfo$stock==stock]) + intyr <- as.numeric(cinfo$intyr[cinfo$stock==stock]) + intbio_low <- as.numeric(cinfo$intbio_low[cinfo$stock==stock]) + intbio_hi <- as.numeric(cinfo$intbio_hi[cinfo$stock==stock]) + endbio_low <- as.numeric(cinfo$endbio_low[cinfo$stock==stock]) + endbio_hi <- as.numeric(cinfo$endbio_hi[cinfo$stock==stock]) + Btype <- as.character(cinfo$Btype[cinfo$stock==stock]) + FutureCrash <- as.character(cinfo$FutureCrash[cinfo$stock==stock]) + comment <- as.character(cinfo$comment[cinfo$stock==stock]) + + + # extract data on stock + yr <- as.numeric(cdat$yr[cdat$stock==stock & cdat$yr >= StartYear & cdat$yr <= EndYear]) + ct <- as.numeric(cdat$ct[cdat$stock==stock & cdat$yr >= StartYear & cdat$yr <= EndYear])/1000 ## assumes that catch is given in tonnes, transforms to '000 tonnes + if(Btype=="observed" | Btype=="CPUE" | Btype=="simulated") { + bt <- as.numeric(cdat$TB[cdat$stock==stock & cdat$yr >= StartYear & cdat$yr <= EndYear])/1000 ## assumes that biomass is in tonnes, transforms to '000 tonnes + } else {bt <- NA} + nyr <- length(yr) # number of years in the time series + + cat("->--------------------------------------- +Species: NA +Name and region: NA , NA +Stock: HLH_M07 +Catch data used from years 1 - 50 +Prior initial relative biomass = 0.5 - 0.9 +Prior intermediate rel. biomass= 0.01 - 0.4 in year 25 +Prior final relative biomass = 0.4 - 0.8 +If current catches continue, is the stock likely to crash within 3 years? No +Prior range for r = 0.2 - 0.8 , prior range for k = 125 - 9965 +Results from Bayesian Schaefer model using catch & biomass ( simulated ) +MSY = 91.7 , 95% CL = 83.9 - 100 +Mean catch / MSY = 0.882 +r = 0.425 , 95% CL = 0.374 - 0.483 +k = 863 , 95% CL = 783 - 951 +Results of CMSY analysis +Altogether 2055 unique viable r-k pairs were found +1142 r-k pairs above the initial geometric mean of r = 0.343 were analysed +r = 0.522 , 95% CL = 0.349 - 0.782 +k = 683 , 95% CL = 438 - 1067 +MSY = 89.2 , 95% CL = 82.2 - 96.7 +Predicted biomass in last year = 0.676 2.5th perc = 0.435 97.5th perc = 0.768 +Predicted biomass in next year = 0.673 2.5th perc = 0.433 97.5th perc = 0.758 +---------------------------------------------------------- +",file=outfile.txt,append=T) + +} + \ No newline at end of file diff --git a/PARALLEL_PROCESSING/DestinationDBHibernate.cfg.xml b/PARALLEL_PROCESSING/DestinationDBHibernate.cfg.xml new file mode 100644 index 0000000..99feeae --- /dev/null +++ b/PARALLEL_PROCESSING/DestinationDBHibernate.cfg.xml @@ -0,0 +1,17 @@ + + + + org.postgresql.Driver + org.hibernate.connection.C3P0ConnectionProvider + jdbc:postgresql://localhost/testdb + gcube + d4science2 + org.hibernate.dialect.PostgreSQLDialect + org.hibernate.transaction.JDBCTransactionFactory + 0 + 1 + 0 + 1 + thread + + \ No newline at end of file diff --git a/PARALLEL_PROCESSING/EcologicalEngineExecutor-1.6.4-SNAPSHOT.jar b/PARALLEL_PROCESSING/EcologicalEngineExecutor-1.6.4-SNAPSHOT.jar new file mode 100644 index 0000000..10d5e95 Binary files /dev/null and b/PARALLEL_PROCESSING/EcologicalEngineExecutor-1.6.4-SNAPSHOT.jar differ diff --git a/PARALLEL_PROCESSING/UpdateLWR_4.R b/PARALLEL_PROCESSING/UpdateLWR_4.R new file mode 100644 index 0000000..9eb7a97 --- /dev/null +++ b/PARALLEL_PROCESSING/UpdateLWR_4.R @@ -0,0 +1,530 @@ +#### R and JAGS code for estimating LWR-parameters from previous studies +#### Meant for updating the ESTIMATE table in FishBase +#### Created by Rainer Froese in March 2013, including JAGS models by James Thorston +#### Modified in June 2013 to include subfamilies + +rm(list=ls(all=TRUE)) # remove previous variables and data +options(digits=3) # 3 significant digits as default +library(R2jags) # Interface with JAGS +runif(1) # sets random seed + +#### Read in data +DataFile = "RF_LWR2.csv" # RF_LWR4 was extracted from FishBase in June 2013 +Data = read.csv(DataFile, header=TRUE) +cat("Start", date(), "\n") +cat("Data file =", DataFile, "\n") +# Get unique, sorted list of Families +Fam.All <- sort(unique(as.character(Data$Family))) +Families <- Fam.All[Fam.All== "Acanthuridae" | Fam.All == "Achiridae"] + +OutFile = "LWR_Test1.csv" +JAGSFILE = "dmnorm_0.bug" + +# Get unique, sorted list of body shapes +Bshape <- sort(unique(as.character(Data$BodyShapeI))) + +#------------------------------------------ +# Functions +#------------------------------------------ + +#--------------------------------------------------------- +# Function to get the priors for the respective body shape +#--------------------------------------------------------- +Get.BS.pr <- function(BS) { + ### Assignment of priors based on available body shape information + # priors derived from 5150 LWR studies in FishBase 02/2013 + + if (BS == "eel-like") { # eel-like prior for log(a) and b + prior_mean_log10a = -2.99 + prior_sd_log10a = 0.175 + prior_tau_log10a = 1/prior_sd_log10a^2 + prior_mean_b = 3.06 + prior_sd_b = 0.0896 + prior_tau_b = 1/prior_sd_b^2 + } else + if (BS == "elongated") { # elongate prior for log(a) and b + prior_mean_log10a = -2.41 + prior_sd_log10a = 0.171 + prior_tau_log10a = 1/prior_sd_log10a^2 + prior_mean_b = 3.12 + prior_sd_b = 0.09 + prior_tau_b = 1/prior_sd_b^2 + } else + if (BS == "fusiform / normal") { # fusiform / normal prior for log(a) and b + prior_mean_log10a = -1.95 + prior_sd_log10a = 0.173 + prior_tau_log10a = 1/prior_sd_log10a^2 + prior_mean_b = 3.04 + prior_sd_b = 0.0857 + prior_tau_b = 1/prior_sd_b^2 + } else + if (BS == "short and / or deep") { # short and / or deep prior for log(a) and b + prior_mean_log10a = -1.7 + prior_sd_log10a = 0.175 + prior_tau_log10a = 1/prior_sd_log10a^2 + prior_mean_b = 3.01 + prior_sd_b = 0.0905 + prior_tau_b = 1/prior_sd_b^2 + } else + # priors across all shapes, used for missing or other BS + { + prior_mean_log10a = -2.0 + prior_sd_log10a = 0.313 + prior_tau_log10a = 1/prior_sd_log10a^2 + prior_mean_b = 3.04 + prior_sd_b = 0.119 + prior_tau_b = 1/prior_sd_b^2 + } + + # Priors for measurement error (= sigma) based on 5150 studies + # given here as shape mu and rate r, for gamma distribution + SD_rObs_log10a = 6520 + SD_muObs_log10a = 25076 + SD_rObs_b = 6808 + SD_muObs_b = 37001 + # Priors for between species variability (= sigma) based on 5150 studies for 1821 species + SD_rGS_log10a = 1372 + SD_muGS_log10a = 7933 + SD_rGS_b = 572 + SD_muGS_b = 6498 + + prior.list <- list(mean_log10a=prior_mean_log10a, sd_log10a=prior_sd_log10a, + tau_log10a=prior_tau_log10a, mean_b=prior_mean_b, sd_b=prior_sd_b, + tau_b=prior_tau_b, SD_rObs_log10a=SD_rObs_log10a, SD_muObs_log10a=SD_muObs_log10a, + SD_rObs_b=SD_rObs_b, SD_muObs_b=SD_muObs_b, SD_rGS_log10a=SD_rGS_log10a, + SD_muGS_log10a=SD_muGS_log10a, SD_rGS_b=SD_rGS_b, SD_muGS_b=SD_muGS_b) +return(prior.list) +} + +#-------------------------------------------------------------------- +# Function to do a Bayesian analysis including LWR from relatives +#-------------------------------------------------------------------- +SpecRelLWR <- function(a, b, wts, GenusSpecies, Nspecies, prior_mean_b, prior_tau_b, + prior_mean_log10a, prior_tau_log10a, SD_rObs_log10a, SD_muObs_log10a, + SD_rObs_b, SD_muObs_b, SD_rGS_log10a, SD_muGS_log10a, + SD_rGS_b, SD_muGS_b){ + ### Define JAGS model + Model = " +model { + #### Process model -- effects of taxonomy + # given the likelihood distributions and the priors, + # create normal posterior distributions for log10a, b, + # and for the process error (=between species variability sigmaGS) + + abTrue[1] ~ dnorm(prior_mean_log10a,prior_tau_log10a) + abTrue[2] ~ dnorm(prior_mean_b,prior_tau_b) + sigmaGSlog10a ~ dgamma( SD_rGS_log10a, SD_muGS_log10a) + sigmaGSb ~ dgamma( SD_rGS_b, SD_muGS_b) + + # given the posterior distributions and the process errors, + # establish for every species the expected witin-species + # parameter distributions; no correlation roGS between species + + roGS <- 0 + tauGenusSpecies[1] <- pow(sigmaGSlog10a,-2) + tauGenusSpecies[2] <- pow(sigmaGSb,-2) + for(k in 1:Nspecies){ + abGenusSpecies[k,1] ~ dnorm(abTrue[1],tauGenusSpecies[1]) + abGenusSpecies[k,2] ~ dnorm(abTrue[2],tauGenusSpecies[2]) + } + + ### Observation model + ## Errors + # given the data and the priors, establish distributions + # for the observation errors sigmaObs + + sigmaObslog10a ~ dgamma( SD_rObs_log10a, SD_muObs_log10a) + sigmaObsb ~ dgamma( SD_rObs_b, SD_muObs_b) + + # create inverse covariance matrix, with negative parameter correlation roObs + roObs ~ dunif(-0.99,0) + CovObs[1,1] <- pow(sigmaObslog10a,2) + CovObs[2,2] <- pow(sigmaObsb,2) + CovObs[1,2] <- roObs * sigmaObslog10a * sigmaObsb + CovObs[2,1] <- CovObs[1,2] + TauObs[1:2,1:2] <- inverse(CovObs[1:2,1:2]) + + ## likelihood + # given the data, the priors and the covariance, + # create multivariate likelihood distributions for log10(a) and b + + for(i in 1:N){ + TauObsI[i,1:2,1:2] <- TauObs[1:2,1:2] * pow(Weights[i],2) # weighted precision + ab[i,1:2] ~ dmnorm(abGenusSpecies[GenusSpecies[i],1:2],TauObsI[i,1:2,1:2]) + } +} + " + + # Write JAGS model + cat(Model, file=JAGSFILE) + # JAGS settings + Nchains = 3 # number of MCMC chains to be used in JAGS + Nburnin = 1e4 # number of burn-in iterations, to be discarded; 1e4 = 10000 iterations for burn-in + Niter = 3e4 # number of iterations after burn-in; 3e4 = 30000 iterations + Nthin = 1e1 # subset of iterations to be used for analysis; 1e1 = every 10th iteration + + # Run JAGS: define data to be passed on in DataJags; + # determine parameters to be returned in Param2Save; + # call JAGS with function Jags() + DataJags = list(ab=cbind(log10(a),b), N=length(a), Weights=wts, Nspecies=Nspecies, GenusSpecies=GenusSpecies, + prior_mean_b=prior_mean_b, prior_tau_b=prior_tau_b, + prior_mean_log10a=prior_mean_log10a, prior_tau_log10a=prior_tau_log10a, + SD_rObs_log10a=SD_rObs_log10a, SD_muObs_log10a=SD_muObs_log10a, + SD_rObs_b=SD_rObs_b, SD_muObs_b=SD_muObs_b, + SD_rGS_log10a=SD_rGS_log10a, SD_muGS_log10a=SD_muGS_log10a, + SD_rGS_b=SD_rGS_b, SD_muGS_b=SD_muGS_b) + Params2Save = c("abTrue","abGenusSpecies","sigmaGSlog10a","sigmaGSb","sigmaObslog10a","sigmaObsb","roObs") + Jags <- jags(inits=NULL, model.file=JAGSFILE, working.directory=NULL, data=DataJags, + parameters.to.save=Params2Save, n.chains=Nchains, n.thin=Nthin, n.iter=Niter, n.burnin=Nburnin) + Jags$BUGSoutput # contains the results from the JAGS run + + # Analyze output for the relatives + abTrue <- Jags$BUGSoutput$sims.list$abTrue + R_mean_log10a <- mean(abTrue[,1]) # true mean of log10(a) + R_sd_log10a <- sd(abTrue[,1]) # true SE of log10(a) + R_mean_b <- mean(abTrue[,2]) # true mean of b + R_sd_b <- sd(abTrue[,2]) # true SE of b + + # Analyze output for the target species + abGenusSpecies <- Jags$BUGSoutput$sims.list$abGenusSpecies + mean_log10a <- mean(abGenusSpecies[,1,1]) # true mean of log10(a) for the first species= target species + sd_log10a <- sd(abGenusSpecies[,1,1]) # true SE of log10(a) + mean_b <- mean(abGenusSpecies[,1,2]) # true mean of b + sd_b <- sd(abGenusSpecies[,1,2]) # true SE of b + mean_sigma_log10a <- mean(Jags$BUGSoutput$sims.list$sigmaObslog10a) # measurement error of log10(a) + sd_sigma_log10a <- apply(as.matrix(Jags$BUGSoutput$sims.list$sigmaObslog10a), 2, sd) + mean_sigma_b <- mean(Jags$BUGSoutput$sims.list$sigmaObsb) # measurement error of b + sd_sigma_b <- apply(as.matrix(Jags$BUGSoutput$sims.list$sigmaObsb), 2, sd) + ro_ab <- mean(Jags$BUGSoutput$sims.list$roObs) # measurement correlation of log10(a),b + + out.list <- list(N=length(a), mean_log10a=mean_log10a, sd_log10a=sd_log10a, mean_b=mean_b, sd_b=sd_b, + R_mean_log10a=R_mean_log10a, R_sd_log10a=R_sd_log10a, R_mean_b=R_mean_b, R_sd_b=R_sd_b) + return(out.list) + } + +#----------------------------------------------------------------------------- +# Function to do a Bayesian LWR analysis with studies for target species only +#----------------------------------------------------------------------------- +SpecLWR <- function(a, b, wts, prior_mean_b, prior_tau_b, + prior_mean_log10a, prior_tau_log10a, SD_rObs_log10a, SD_muObs_log10a, + SD_rObs_b, SD_muObs_b, SD_rGS_log10a, SD_muGS_log10a, + SD_rGS_b, SD_muGS_b){ + + # Define JAGS model + Model = " + model { + sigma1 ~ dgamma( SD_rObs_log10a, SD_muObs_log10a) # posterior distribution for measurement error in log10a + sigma2 ~ dgamma( SD_rObs_b, SD_muObs_b) # posterior distribution for measurement error in log10a + + ro ~ dunif(-0.99,0) # uniform prior for negative correlation between log10a and b + abTrue[1] ~ dnorm(prior_mean_log10a,prior_tau_log10a) # normal posterior distribution for log10a + abTrue[2] ~ dnorm(prior_mean_b,prior_tau_b) # normal posterior distribution for b + CovObs[1,1] <- pow(sigma1,2) + CovObs[2,2] <- pow(sigma2,2) + CovObs[1,2] <- ro * sigma1 * sigma2 + CovObs[2,1] <- CovObs[1,2] + TauObs[1:2,1:2] <- inverse(CovObs[1:2,1:2]) # create inverse covariance matrix + for(i in 1:N){ + TauObsI[i,1:2,1:2] <- TauObs[1:2,1:2] * pow(Weights[i],2) # converts prior SD into prior weighted precision + + # given the data, the priors and the covariance, create multivariate normal posteriors for log(a) and b + ab[i,1:2] ~ dmnorm(abTrue[1:2],TauObsI[i,1:2,1:2]) + } + } + " + +# Write JAGS model +cat(Model, file=JAGSFILE) +# JAGS settings +Nchains = 3 # number of MCMC chains to be used in JAGS +Nburnin = 1e4 # number of burn-in runs, to be discarded; 10000 iterations for burn-in +Niter = 3e4 # number of iterations after burn-in; 3e4 = 30000 iterations +Nthin = 1e1 # subset of iterations to be used for analysis; 1e1 = every 10th iteration +# Run JAGS: define data to be passed on in DataJags; determine parameters to be returned in Param2Save; call JAGS with function Jags() +DataJags = list(ab=cbind(log10(a),b), N=length(a), Weights=wts, prior_mean_b=prior_mean_b, + prior_tau_b=prior_tau_b, prior_mean_log10a=prior_mean_log10a, prior_tau_log10a=prior_tau_log10a, + SD_rObs_log10a=SD_rObs_log10a, SD_muObs_log10a=SD_muObs_log10a, + SD_rObs_b=SD_rObs_b, SD_muObs_b=SD_muObs_b) +Params2Save = c("abTrue","sigma1","sigma2","ro") +Jags <- jags(inits=NULL, model.file=JAGSFILE, working.directory=NULL, data=DataJags, parameters.to.save=Params2Save, n.chains=Nchains, n.thin=Nthin, n.iter=Niter, n.burnin=Nburnin) +Jags$BUGSoutput # contains the results from the JAGS run +# Analyze output +abTrue <- Jags$BUGSoutput$sims.list$abTrue +mean_log10a <- mean(abTrue[,1]) # true mean of log10(a) +sd_log10a <- sd(abTrue[,1]) # true SE of log10(a) +mean_b <- mean(abTrue[,2]) # true mean of b +sd_b <- sd(abTrue[,2]) # true SE of b +mean_sigma_log10a <- mean(Jags$BUGSoutput$sims.list$sigma1) # measurement error of log10(a) +sd_sigma_log10a <- apply(as.matrix(Jags$BUGSoutput$sims.list$sigma1), 2, sd) +mean_sigma_b <- mean(Jags$BUGSoutput$sims.list$sigma2) # measurement error of b +sd_sigma_b <- apply(as.matrix(Jags$BUGSoutput$sims.list$sigma2), 2, sd) +ro_ab <- mean(Jags$BUGSoutput$sims.list$ro) # measurement correlation of log10(a),b + +out.list <- list(N=length(a), mean_log10a=mean_log10a, sd_log10a=sd_log10a, mean_b=mean_b, sd_b=sd_b) +return(out.list) + +} # End of Functions section + +#-------------------------------- +# Analysis by Family +#-------------------------------- +# Do LWR analysis by Family, Subfamily and Body shape, depending on available LWR studies +# for(Fam in "Acanthuridae") { +for(Fam in Families) { + Subfamilies <- sort(unique(Data$Subfamily[Data$Family==Fam])) + for(SF in Subfamilies) { + for(BS in Bshape) { + # get species (SpecCodes) in this Subfamily and with this body shape + SpecCode.SF.BS <- unique(Data$SpecCode[Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS]) + # if there are species with this body shape + if(length(SpecCode.SF.BS > 0)) { + # get priors for this body shape + prior <- Get.BS.pr(BS) + # get LWR data for this body shape + b_raw <- Data$b[Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS] + cat("\n") + cat("Family =", Fam, ", Subfamily =", SF, ", Body shape =", BS, ", Species =", length(SpecCode.SF.BS), ", LWR =", + length(b_raw[is.na(b_raw)==F]), "\n") + # if no LWR studies exist for this body shape, assign the respective priors to all species + if(length(b_raw[is.na(b_raw)==F])==0) { + # assign priors to species with no LWR in this Subfamily with this body shape + cat("Assigning overall body shape prior to", length(SpecCode.SF.BS), " species \n") + for(SpC in SpecCode.SF.BS) { + out.prior <- data.frame(Fam, SF, BS, SpC, 0, prior$mean_log10a, prior$sd_log10a, prior$mean_b, prior$sd_b, + paste("all LWR estimates for this BS")) + write.table(out.prior, file=OutFile, append = T, sep=",", dec=".", row.names=F, col.names=F) + } + } else { + + # Update priors for this body shape using existing LWR studies + # get LWR data for this Subfamily and body shape + Keep <- which(Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS & is.na(Data$b)==F & Data$Score>0) + wts <- Data$Score[Keep] # Un-normalized weights (so that Cov is comparable among analyses) + a <- Data$a[Keep] + b <- Data$b[Keep] + GenSpec <- paste(Data$Genus[Keep],Data$Species[Keep]) + # add a first dummy record with prior LWR and low score = 0.3, as pseudo target species + # Name of dummy target species is Dum1 dum1 + TargetSpec = paste("Dum1", "dum1") + wts <- c(0.3, wts) + a <- c(10^(prior$mean_log10a), a) + b <- c(prior$mean_b, b) + GenSpec <- c(TargetSpec, GenSpec) + # Relabel GenSpec so that TargetSpec = level 1 + OtherSpecies = unique(GenSpec[GenSpec != TargetSpec]) + GenusSpecies = factor(GenSpec, levels=c(TargetSpec, OtherSpecies)) + Nspecies = nlevels(GenusSpecies) # number of species + # run Bayesian analysis for pseudo target species with Subfamily members + # The resulting R_mean_log10a, R_sd_log10a, R_mean_b, R_sd_b will be used for species without LWR + cat("Updating Subfamily-Bodyshape prior using", Nspecies-1, "species with LWR studies \n") + prior.SFam.BS <- SpecRelLWR(a, b, wts, GenusSpecies, Nspecies, prior_mean_b=prior$mean_b, + prior_tau_b=prior$tau_b, prior_mean_log10a=prior$mean_log10a, + prior_tau_log10a=prior$tau_log10a, SD_rObs_log10a=prior$SD_rObs_log10a, + SD_muObs_log10a=prior$SD_muObs_log10a, SD_rObs_b=prior$SD_rObs_b, + SD_muObs_b=prior$SD_muObs_b, SD_rGS_log10a=prior$SD_rGS_log10a, + SD_muGS_log10a=prior$SD_muGS_log10a, SD_rGS_b=prior$SD_rGS_b, + SD_muGS_b=prior$SD_muGS_b) + + #------------------------------------------------------------------------------------------ + # if there are Genera with >= 5 species with LWR, update body shape priors for these Genera + #------------------------------------------------------------------------------------------ + Genera <- unique(as.character(Data$Genus[Keep])) + # create empty list of lists for storage of generic priors + prior.Gen.BS <- rep(list(list()),length(Genera)) # create a list of empty lists + names(prior.Gen.BS) <- Genera # name the list elements according to the Genera + for(Genus in Genera){ + # check if Genus contains >= 5 species with LWR data + if(length(unique(Data$SpecCode[Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS & is.na(Data$b)==F & + Data$Score>0 & Data$Genus==Genus]))>=5) { + # run Subfamily analysis with only data for this genus + Keep <- which(Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS & is.na(Data$b)==F & Data$Score>0 & + Data$Genus==Genus) + wts <- Data$Score[Keep] # Un-normalized weights (so that Cov is comparable among analyses) + a <- Data$a[Keep] + b <- Data$b[Keep] + GenSpec <- paste(Data$Genus[Keep],Data$Species[Keep]) + # add a first dummy record with prior LWR and low score = 0.3, as pseudo target species + # Name of dummy target species is Dum1 dum1 + TargetSpec = paste("Dum1", "dum1") + wts <- c(0.3, wts) + a <- c(10^(prior$mean_log10a), a) + b <- c(prior$mean_b, b) + GenSpec <- c(TargetSpec, GenSpec) + # Relabel GenSpec so that TargetSpec = level 1 + OtherSpecies = unique(GenSpec[GenSpec != TargetSpec]) + GenusSpecies = factor(GenSpec, levels=c(TargetSpec, OtherSpecies)) + Nspecies = nlevels(GenusSpecies) # number of species + # run Bayesian analysis for pseudo target species with Genus members + # R_mean_log10a, R_sd_log10a, R_mean_b, R_sd_b will be used for species without LWR + cat("Updating prior for Genus =", Genus, ", with", Nspecies -1, "LWR Species \n") + prior.Gen.BS[[Genus]] <- SpecRelLWR(a, b, wts, GenusSpecies, Nspecies, + prior_mean_b=prior.SFam.BS$R_mean_b, + prior_tau_b=1/prior.SFam.BS$R_sd_b^2, + prior_mean_log10a=prior.SFam.BS$R_mean_log10a, + prior_tau_log10a=1/prior.SFam.BS$R_sd_log10a, + SD_rObs_log10a=prior$SD_rObs_log10a, + SD_muObs_log10a=prior$SD_muObs_log10a, SD_rObs_b=prior$SD_rObs_b, + SD_muObs_b=prior$SD_muObs_b, SD_rGS_log10a=prior$SD_rGS_log10a, + SD_muGS_log10a=prior$SD_muGS_log10a, SD_rGS_b=prior$SD_rGS_b, + SD_muGS_b=prior$SD_muGS_b) + } + } + # new Subfamily-BS priors have been generated + # for some genera, new Genus-BS priors have been generated + # --------------------------------------------------------------------- + # Loop through all species in this Subfamily-BS; assign LWR as appropriate + # --------------------------------------------------------------------- + for(SpC in SpecCode.SF.BS) { + Genus <- as.character(unique(Data$Genus[Data$SpecCode==SpC])) + Species <- as.character(unique(Data$Species[Data$SpecCode==SpC])) + TargetSpec = paste(Genus, Species) + LWR <- length(Data$b[Data$SpecCode==SpC & is.na(Data$b)==F & Data$Score>0]) + LWRGenspec <- length(unique(Data$SpecCode[Data$BodyShapeI==BS & is.na(Data$b)==F & + Data$Score>0 & Data$Genus==Genus])) + LWRSFamspec <- length(unique(Data$SpecCode[Data$BodyShapeI==BS & is.na(Data$b)==F & + Data$Score>0 & Data$Family==Fam & Data$Subfamily==SF])) + #--------------------------------------------------------- + # >= 5 LWR in target species, run single species analysis + if(LWR >= 5) { + # Run analysis with data only for this species + Keep <- which(Data$SpecCode==SpC & is.na(Data$b)==F & Data$Score>0) + wts = Data$Score[Keep] # Un-normalized weights (so that Cov is comparable among analyses) + a = Data$a[Keep] + b = Data$b[Keep] + + # determine priors to be used + if(LWRGenspec >= 5) { + prior_mean_b=prior.Gen.BS[[Genus]]$R_mean_b + prior_tau_b=1/prior.Gen.BS[[Genus]]$R_sd_b^2 + prior_mean_log10a=prior.Gen.BS[[Genus]]$R_mean_log10a + prior_tau_log10a=1/prior.Gen.BS[[Genus]]$R_sd_log10a^2 + } else + if (LWRSFamspec > 0) { + prior_mean_b=prior.SFam.BS$R_mean_b + prior_tau_b=1/prior.SFam.BS$R_sd_b^2 + prior_mean_log10a=prior.SFam.BS$R_mean_log10a + prior_tau_log10a=1/prior.SFam.BS$R_sd_log10a^2 + } else { + prior_mean_b=prior$mean_b + prior_tau_b=prior$tau_b + prior_mean_log10a=prior$mean_log10a + prior_tau_log10a=prior$tau_log10a + } + cat("Running single species analysis for", TargetSpec, "LWR =", LWR, ", LWR species in Genus=",LWRGenspec,"\n" ) + # call function for single species analysis + post <- SpecLWR(a, b, wts, prior_mean_b=prior_mean_b, + prior_tau_b=prior_tau_b, prior_mean_log10a=prior_mean_log10a, + prior_tau_log10a=prior_tau_log10a, SD_rObs_log10a=prior$SD_rObs_log10a, + SD_muObs_log10a=prior$SD_muObs_log10a, SD_rObs_b=prior$SD_rObs_b, + SD_muObs_b=prior$SD_muObs_b, SD_rGS_log10a=prior$SD_rGS_log10a, + SD_muGS_log10a=prior$SD_muGS_log10a, SD_rGS_b=prior$SD_rGS_b, + SD_muGS_b=prior$SD_muGS_b) + out.SpC <- data.frame(Fam, SF, BS, SpC, LWR, format(post$mean_log10a, digits=3), format(post$sd_log10a, digits=3), format(post$mean_b, disgits=3), format(post$sd_b, digits=3), + paste("LWR estimates for this species")) + write.table(out.SpC, file=OutFile, append = T, sep=",", dec=".", row.names=F, col.names=F) + + } else + #-------------------------------------------------------- + # 1-4 LWR in target species and >= 5 LWR species in Genus + # run hierarchical analysis for genus members, with Subfamily-BS prior + if(LWR >= 1 & LWRGenspec >=5) { + # run Subfamily analysis with only data for this genus + Keep <- which(Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS & is.na(Data$b)==F & Data$Score>0 & + Data$Genus==Genus) + wts <- Data$Score[Keep] # Un-normalized weights (so that Cov is comparable among analyses) + a <- Data$a[Keep] + b <- Data$b[Keep] + GenSpec <- paste(Data$Genus[Keep],Data$Species[Keep]) + + # Relabel GenSpec so that TargetSpec = level 1 + OtherSpecies = unique(GenSpec[GenSpec != TargetSpec]) + GenusSpecies = factor(GenSpec, levels=c(TargetSpec, OtherSpecies)) + Nspecies = nlevels(GenusSpecies) # number of species + # run Bayesian analysis for target species with Genus members + cat("Running analysis with congeners for", TargetSpec, ", LWR =", LWR,", LWR species in Genus =", LWRGenspec,"\n") + post <- SpecRelLWR(a, b, wts, GenusSpecies, Nspecies, + prior_mean_b=prior.SFam.BS$R_mean_b, + prior_tau_b=1/prior.SFam.BS$R_sd_b^2, + prior_mean_log10a=prior.SFam.BS$R_mean_log10a, + prior_tau_log10a=1/prior.SFam.BS$R_sd_log10a^2, + SD_rObs_log10a=prior$SD_rObs_log10a, + SD_muObs_log10a=prior$SD_muObs_log10a, SD_rObs_b=prior$SD_rObs_b, + SD_muObs_b=prior$SD_muObs_b, SD_rGS_log10a=prior$SD_rGS_log10a, + SD_muGS_log10a=prior$SD_muGS_log10a, SD_rGS_b=prior$SD_rGS_b, + SD_muGS_b=prior$SD_muGS_b) + out.SpC <- data.frame(Fam, SF, BS, SpC, LWR, format(post$mean_log10a, digits=3), format(post$sd_log10a, digits=3), format(post$mean_b, disgits=3), format(post$sd_b, digits=3), + paste("LWR estimates for species & Genus-BS")) + write.table(out.SpC, file=OutFile, append = T, sep=",", dec=".", row.names=F, col.names=F) + } else + + #------------------------------------------------------- + # 1-4 LWR in target species and < 5 LWR species in Genus + # run hierarchical analysis for Subfamily members, with bodyshape prior + + if(LWR >= 1 & LWRSFamspec > 1) { + # run Subfamily analysis + Keep <- which(Data$Family==Fam & Data$Subfamily==SF & Data$BodyShapeI==BS & is.na(Data$b)==F & Data$Score>0) + wts <- Data$Score[Keep] # Un-normalized weights (so that Cov is comparable among analyses) + a <- Data$a[Keep] + b <- Data$b[Keep] + GenSpec <- paste(Data$Genus[Keep],Data$Species[Keep]) + # Relabel GenSpec so that TargetSpec = level 1 + OtherSpecies = unique(GenSpec[GenSpec != TargetSpec]) + GenusSpecies = factor(GenSpec, levels=c(TargetSpec, OtherSpecies)) + Nspecies = nlevels(GenusSpecies) # number of species + # run Bayesian analysis for target species with Subfamily members + cat("Running analysis with Subfamily members for", TargetSpec, ", LWR =", LWR,", LWR species in Subfamily-BS =", + LWRSFamspec, "\n") + post <- SpecRelLWR(a, b, wts, GenusSpecies, Nspecies, + prior_mean_b=prior$mean_b, + prior_tau_b=prior$tau_b, + prior_mean_log10a=prior$mean_log10a, + prior_tau_log10a=prior$tau_log10a, + SD_rObs_log10a=prior$SD_rObs_log10a, + SD_muObs_log10a=prior$SD_muObs_log10a, SD_rObs_b=prior$SD_rObs_b, + SD_muObs_b=prior$SD_muObs_b, SD_rGS_log10a=prior$SD_rGS_log10a, + SD_muGS_log10a=prior$SD_muGS_log10a, SD_rGS_b=prior$SD_rGS_b, + SD_muGS_b=prior$SD_muGS_b) + out.SpC <- data.frame(Fam, SF, BS, SpC, LWR, format(post$mean_log10a, digits=3), format(post$sd_log10a, digits=3), + format(post$mean_b, disgits=3), format(post$sd_b, digits=3), + paste("LWR estimates for species & Subfamily-BS")) + write.table(out.SpC, file=OutFile, append = T, sep=",", dec=".", row.names=F, col.names=F) + } else + #-------------------------------------------------- + # assign Genus-BS priors to target species + if(LWRGenspec >= 5) { + cat("Assign Genus-BS prior for", TargetSpec, "\n") + out.SpC <- data.frame(Fam, SF, BS, SpC, LWR, format(prior.Gen.BS[[Genus]]$mean_log10a, digits=3), + format(prior.Gen.BS[[Genus]]$sd_log10a, digits=3), + format(prior.Gen.BS[[Genus]]$mean_b, digits=3), format(prior.Gen.BS[[Genus]]$sd_b, digits=3), + paste("LWR estimates for this Genus-BS")) + write.table(out.SpC, file=OutFile, append = T, sep=",", dec=".", row.names=F, col.names=F) + } else { + # ----------------------------------------------- + # assign Subfamily-BS priors to target species + cat("Assign Subfamily-BS prior for", TargetSpec,"\n") + out.SpC <- data.frame(Fam, SF, BS, SpC, LWR, format(prior.SFam.BS$mean_log10a, digits=3), format(prior.SFam.BS$sd_log10a, digits=3), + format(prior.SFam.BS$mean_b, digits=3), format(prior.SFam.BS$sd_b, digits=3), paste("LWR estimates for this Subfamily-BS")) + write.table(out.SpC, file=OutFile, append = T, sep=",", dec=".", row.names=F, col.names=F) + } + } # end of species loop for this Subfamily and body shape + + } # end of section dealing with Subfamily - body shapes that contain LWR estimates + + } # end of section that deals with Subfamily - body shapes that contain species + + } # end of body shape section + + } # end of Subfamily section + +} # end of Family section +cat("End", date(),"\n") + + + + + + + + + diff --git a/PARALLEL_PROCESSING/YASMEEN-matcher-1.2.0.1.jar b/PARALLEL_PROCESSING/YASMEEN-matcher-1.2.0.1.jar new file mode 100644 index 0000000..ef14b6f Binary files /dev/null and b/PARALLEL_PROCESSING/YASMEEN-matcher-1.2.0.1.jar differ diff --git a/PARALLEL_PROCESSING/YASMEEN-parser-1.2.0.jar b/PARALLEL_PROCESSING/YASMEEN-parser-1.2.0.jar new file mode 100644 index 0000000..42e28c2 Binary files /dev/null and b/PARALLEL_PROCESSING/YASMEEN-parser-1.2.0.jar differ diff --git a/PARALLEL_PROCESSING/c3p0-0.9.1.2.jar b/PARALLEL_PROCESSING/c3p0-0.9.1.2.jar new file mode 100644 index 0000000..0f42d60 Binary files /dev/null and b/PARALLEL_PROCESSING/c3p0-0.9.1.2.jar differ diff --git a/PARALLEL_PROCESSING/common-configuration-scanner-1.0.1-SNAPSHOT.jar b/PARALLEL_PROCESSING/common-configuration-scanner-1.0.1-SNAPSHOT.jar new file mode 100644 index 0000000..d5c087d Binary files /dev/null and b/PARALLEL_PROCESSING/common-configuration-scanner-1.0.1-SNAPSHOT.jar differ diff --git a/PARALLEL_PROCESSING/common-encryption-1.0.1-3.5.0.jar b/PARALLEL_PROCESSING/common-encryption-1.0.1-3.5.0.jar new file mode 100644 index 0000000..20f4a73 Binary files /dev/null and b/PARALLEL_PROCESSING/common-encryption-1.0.1-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/common-gcore-resources-1.2.0-3.5.0.jar b/PARALLEL_PROCESSING/common-gcore-resources-1.2.0-3.5.0.jar new file mode 100644 index 0000000..27a0ec5 Binary files /dev/null and b/PARALLEL_PROCESSING/common-gcore-resources-1.2.0-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/common-gcore-stubs-1.2.0-3.5.0.jar b/PARALLEL_PROCESSING/common-gcore-stubs-1.2.0-3.5.0.jar new file mode 100644 index 0000000..caf569a Binary files /dev/null and b/PARALLEL_PROCESSING/common-gcore-stubs-1.2.0-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/common-scope-1.2.1-SNAPSHOT.jar b/PARALLEL_PROCESSING/common-scope-1.2.1-SNAPSHOT.jar new file mode 100644 index 0000000..2602d3e Binary files /dev/null and b/PARALLEL_PROCESSING/common-scope-1.2.1-SNAPSHOT.jar differ diff --git a/PARALLEL_PROCESSING/common-scope-maps-1.0.2-3.5.0.jar b/PARALLEL_PROCESSING/common-scope-maps-1.0.2-3.5.0.jar new file mode 100644 index 0000000..e1ee05d Binary files /dev/null and b/PARALLEL_PROCESSING/common-scope-maps-1.0.2-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/commons-collections-3.1.jar b/PARALLEL_PROCESSING/commons-collections-3.1.jar new file mode 100644 index 0000000..41e230f Binary files /dev/null and b/PARALLEL_PROCESSING/commons-collections-3.1.jar differ diff --git a/PARALLEL_PROCESSING/commons-io-1.2.jar b/PARALLEL_PROCESSING/commons-io-1.2.jar new file mode 100644 index 0000000..b2867cd Binary files /dev/null and b/PARALLEL_PROCESSING/commons-io-1.2.jar differ diff --git a/PARALLEL_PROCESSING/discovery-client-1.0.1-3.5.0.jar b/PARALLEL_PROCESSING/discovery-client-1.0.1-3.5.0.jar new file mode 100644 index 0000000..ea61c43 Binary files /dev/null and b/PARALLEL_PROCESSING/discovery-client-1.0.1-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/dom4j-1.6.1.jar b/PARALLEL_PROCESSING/dom4j-1.6.1.jar new file mode 100644 index 0000000..c8c4dbb Binary files /dev/null and b/PARALLEL_PROCESSING/dom4j-1.6.1.jar differ diff --git a/PARALLEL_PROCESSING/ecological-engine-1.8.1-SNAPSHOT.jar b/PARALLEL_PROCESSING/ecological-engine-1.8.1-SNAPSHOT.jar new file mode 100644 index 0000000..1d8beea Binary files /dev/null and b/PARALLEL_PROCESSING/ecological-engine-1.8.1-SNAPSHOT.jar differ diff --git a/PARALLEL_PROCESSING/hibernate3.jar b/PARALLEL_PROCESSING/hibernate3.jar new file mode 100644 index 0000000..f97dcdb Binary files /dev/null and b/PARALLEL_PROCESSING/hibernate3.jar differ diff --git a/PARALLEL_PROCESSING/ic-client-1.0.1-3.5.0.jar b/PARALLEL_PROCESSING/ic-client-1.0.1-3.5.0.jar new file mode 100644 index 0000000..792b11f Binary files /dev/null and b/PARALLEL_PROCESSING/ic-client-1.0.1-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/jaxen-1.1.2.jar b/PARALLEL_PROCESSING/jaxen-1.1.2.jar new file mode 100644 index 0000000..69de309 Binary files /dev/null and b/PARALLEL_PROCESSING/jaxen-1.1.2.jar differ diff --git a/PARALLEL_PROCESSING/jta-1.1.jar b/PARALLEL_PROCESSING/jta-1.1.jar new file mode 100644 index 0000000..6d225b7 Binary files /dev/null and b/PARALLEL_PROCESSING/jta-1.1.jar differ diff --git a/PARALLEL_PROCESSING/log4j-1.2.16.jar b/PARALLEL_PROCESSING/log4j-1.2.16.jar new file mode 100644 index 0000000..3f9d847 Binary files /dev/null and b/PARALLEL_PROCESSING/log4j-1.2.16.jar differ diff --git a/PARALLEL_PROCESSING/mongo-java-driver-2.12.4.jar b/PARALLEL_PROCESSING/mongo-java-driver-2.12.4.jar new file mode 100644 index 0000000..ecbaecd Binary files /dev/null and b/PARALLEL_PROCESSING/mongo-java-driver-2.12.4.jar differ diff --git a/PARALLEL_PROCESSING/postgresql-8.4-702.jdbc4.jar b/PARALLEL_PROCESSING/postgresql-8.4-702.jdbc4.jar new file mode 100644 index 0000000..8b0c761 Binary files /dev/null and b/PARALLEL_PROCESSING/postgresql-8.4-702.jdbc4.jar differ diff --git a/PARALLEL_PROCESSING/slf4j-api-1.6.0.jar b/PARALLEL_PROCESSING/slf4j-api-1.6.0.jar new file mode 100644 index 0000000..db92f9a Binary files /dev/null and b/PARALLEL_PROCESSING/slf4j-api-1.6.0.jar differ diff --git a/PARALLEL_PROCESSING/slf4j-log4j12-1.6.0.jar b/PARALLEL_PROCESSING/slf4j-log4j12-1.6.0.jar new file mode 100644 index 0000000..42cca51 Binary files /dev/null and b/PARALLEL_PROCESSING/slf4j-log4j12-1.6.0.jar differ diff --git a/PARALLEL_PROCESSING/storage-manager-core-2.1.2-3.5.0.jar b/PARALLEL_PROCESSING/storage-manager-core-2.1.2-3.5.0.jar new file mode 100644 index 0000000..1202e26 Binary files /dev/null and b/PARALLEL_PROCESSING/storage-manager-core-2.1.2-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/storage-manager-wrapper-2.1.0-3.5.0.jar b/PARALLEL_PROCESSING/storage-manager-wrapper-2.1.0-3.5.0.jar new file mode 100644 index 0000000..24eec7a Binary files /dev/null and b/PARALLEL_PROCESSING/storage-manager-wrapper-2.1.0-3.5.0.jar differ diff --git a/PARALLEL_PROCESSING/xalan-2.6.0.jar b/PARALLEL_PROCESSING/xalan-2.6.0.jar new file mode 100644 index 0000000..73cf175 Binary files /dev/null and b/PARALLEL_PROCESSING/xalan-2.6.0.jar differ diff --git a/PARALLEL_PROCESSING/xpp3_min-1.1.4c.jar b/PARALLEL_PROCESSING/xpp3_min-1.1.4c.jar new file mode 100644 index 0000000..813a9a8 Binary files /dev/null and b/PARALLEL_PROCESSING/xpp3_min-1.1.4c.jar differ diff --git a/PARALLEL_PROCESSING/xstream-1.3.1.jar b/PARALLEL_PROCESSING/xstream-1.3.1.jar new file mode 100644 index 0000000..4ef4219 Binary files /dev/null and b/PARALLEL_PROCESSING/xstream-1.3.1.jar differ diff --git a/aquamapsjarcreator.jardesc b/aquamapsjarcreator.jardesc new file mode 100644 index 0000000..e2111ef --- /dev/null +++ b/aquamapsjarcreator.jardesc @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/cfg/ALog.properties b/cfg/ALog.properties new file mode 100644 index 0000000..cd47834 --- /dev/null +++ b/cfg/ALog.properties @@ -0,0 +1,32 @@ +#### Use two appenders, one to log to console, another to log to a file +log4j.rootCategory= R + +#### First appender writes to console +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{dd/MM/yyyy HH:mm:ss} %p %t %c - %m%n +#log4j.appender.stdout.layout.ConversionPattern=%m%n +#log4j.appender.stdout.File=Analysis.log + +#### Second appender writes to a file +log4j.logger.AnalysisLogger=trace,stdout, R +log4j.appender.R=org.apache.log4j.RollingFileAppender +#log4j.appender.R=org.apache.log4j.AsyncAppender +#log4j.appender.R.Threshold=INFO +log4j.appender.R.File=Analysis.log +log4j.appender.R.MaxFileSize=50000KB +log4j.appender.R.MaxBackupIndex=2 +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d{dd/MM/yyyy HH:mm:ss} %p %t %c - %m%n +#log4j.appender.R.layout.ConversionPattern=%m%n + +#### Third appender writes to a file +log4j.logger.org.hibernate=H +#log4j.appender.H=org.apache.log4j.RollingFileAppender +log4j.appender.H=org.apache.log4j.AsyncAppender +#log4j.appender.H.File=HibernateLog.log +#log4j.appender.H.MaxFileSize=1024KB +#log4j.appender.H.MaxBackupIndex=2 +log4j.appender.H.layout=org.apache.log4j.PatternLayout +log4j.appender.H.layout.ConversionPattern=%d{dd/MM/yyyy HH:mm:ss} %p %t %c - %m%n + diff --git a/cfg/DestinationDBHibernate.cfg.out.xml b/cfg/DestinationDBHibernate.cfg.out.xml new file mode 100644 index 0000000..e69de29 diff --git a/cfg/DestinationDBHibernate.cfg.xml b/cfg/DestinationDBHibernate.cfg.xml new file mode 100644 index 0000000..28b14d3 --- /dev/null +++ b/cfg/DestinationDBHibernate.cfg.xml @@ -0,0 +1,18 @@ + + + + org.postgresql.Driver + org.hibernate.connection.C3P0ConnectionProvider + jdbc:postgresql://localhost/testdb + gcube + d4science2 + + org.hibernate.dialect.PostgreSQLDialect + org.hibernate.transaction.JDBCTransactionFactory + 0 + 10 + 0 + 1 + thread + + \ No newline at end of file diff --git a/cfg/algorithms.properties b/cfg/algorithms.properties new file mode 100644 index 0000000..a8d3b68 --- /dev/null +++ b/cfg/algorithms.properties @@ -0,0 +1,9 @@ +AQUAMAPS_SUITABLE=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsSuitable +AQUAMAPS_NATIVE=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNative +AQUAMAPS_NATIVE_2050=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNative2050 +AQUAMAPS_SUITABLE_2050=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsSuitable2050 +AQUAMAPS_NATIVE_NEURALNETWORK=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNN +AQUAMAPS_SUITABLE_NEURALNETWORK=org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNNSuitable +FEED_FORWARD_A_N_N_DISTRIBUTION=org.gcube.dataanalysis.ecoengine.spatialdistributions.FeedForwardNeuralNetworkDistribution +LWR=org.gcube.dataanalysis.executor.nodes.algorithms.LWR +CMSY=org.gcube.dataanalysis.executor.nodes.algorithms.CMSY \ No newline at end of file diff --git a/cfg/clusterers.properties b/cfg/clusterers.properties new file mode 100644 index 0000000..c2801e9 --- /dev/null +++ b/cfg/clusterers.properties @@ -0,0 +1,3 @@ +DBSCAN=org.gcube.dataanalysis.ecoengine.clustering.DBScan +KMEANS=org.gcube.dataanalysis.ecoengine.clustering.KMeans +XMEANS=org.gcube.dataanalysis.ecoengine.clustering.XMeansWrapper \ No newline at end of file diff --git a/cfg/evaluators.properties b/cfg/evaluators.properties new file mode 100644 index 0000000..1245533 --- /dev/null +++ b/cfg/evaluators.properties @@ -0,0 +1,3 @@ +DISCREPANCY_ANALYSIS=org.gcube.dataanalysis.ecoengine.evaluation.DiscrepancyAnalysis +QUALITY_ANALYSIS=org.gcube.dataanalysis.ecoengine.evaluation.DistributionQualityAnalysis +HRS=org.gcube.dataanalysis.ecoengine.evaluation.HabitatRepresentativeness \ No newline at end of file diff --git a/cfg/generators.properties b/cfg/generators.properties new file mode 100644 index 0000000..3f13c9c --- /dev/null +++ b/cfg/generators.properties @@ -0,0 +1,6 @@ +LOCAL_WITH_DATABASE=org.gcube.dataanalysis.ecoengine.processing.LocalSplitGenerator +SIMPLE_LOCAL=org.gcube.dataanalysis.ecoengine.processing.LocalSimpleSplitGenerator +D4SCIENCE=org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing +#OCCURRENCES_MERGER=org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing +OCCURRENCES_INTERSECTOR=org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing +#OCCURRENCES_SUBTRACTION=org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing diff --git a/cfg/interpolateTacsat.r b/cfg/interpolateTacsat.r new file mode 100644 index 0000000..a876861 --- /dev/null +++ b/cfg/interpolateTacsat.r @@ -0,0 +1,439 @@ +cat("Retrieving Input Parameters\n") +inputFile<-'tacsat.csv' +outputFile<-'tacsat_interpolated.csv' + +require(data.table) +print(Sys.time()) + +memory.size(max = TRUE) +memory.limit(size = 4000) + +interCubicHermiteSpline <- function(spltx,spltCon,res,params,headingAdjustment){ + + #Formula of Cubic Hermite Spline + t <- seq(0,1,length.out=res) + F00 <- 2*t^3 -3*t^2 + 1 + F10 <- t^3-2*t^2+t + F01 <- -2*t^3+3*t^2 + F11 <- t^3-t^2 + + #Making tacsat dataset ready + spltx[spltCon[,1],"SI_HE"][which(is.na(spltx[spltCon[,1],"SI_HE"]))] <- 0 + spltx[spltCon[,2],"SI_HE"][which(is.na(spltx[spltCon[,2],"SI_HE"]))] <- 0 + + #Heading at begin point in degrees + Hx0 <- sin(spltx[spltCon[,1],"SI_HE"]/(180/pi)) + Hy0 <- cos(spltx[spltCon[,1],"SI_HE"]/(180/pi)) + + #Heading at end point in degrees + Hx1 <- sin(spltx[spltCon[,2]-headingAdjustment,"SI_HE"]/(180/pi)) + Hy1 <- cos(spltx[spltCon[,2]-headingAdjustment,"SI_HE"]/(180/pi)) + + #Start and end positions + Mx0 <- spltx[spltCon[,1],"SI_LONG"] + Mx1 <- spltx[spltCon[,2],"SI_LONG"] + My0 <- spltx[spltCon[,1],"SI_LATI"] + My1 <- spltx[spltCon[,2],"SI_LATI"] + + #Corrected for longitude lattitude effect + Hx0 <- Hx0 * params$fm * spltx[spltCon[,1],"SI_SP"] /((params$st[2]-params$st[1])/2+params$st[1]) + Hx1 <- Hx1 * params$fm * spltx[spltCon[,2],"SI_SP"] /((params$st[2]-params$st[1])/2+params$st[1]) + Hy0 <- Hy0 * params$fm * lonLatRatio(spltx[spltCon[,1],"SI_LONG"],spltx[spltCon[,1],"SI_LATI"]) * spltx[spltCon[,1],"SI_SP"]/((params$st[2]-params$st[1])/2+params$st[1]) + Hy1 <- Hy1 * params$fm * lonLatRatio(spltx[spltCon[,2],"SI_LONG"],spltx[spltCon[,2],"SI_LATI"]) * spltx[spltCon[,2],"SI_SP"]/((params$st[2]-params$st[1])/2+params$st[1]) + + #Get the interpolation + fx <- outer(F00,Mx0,"*")+outer(F10,Hx0,"*")+outer(F01,Mx1,"*")+outer(F11,Hx1,"*") + fy <- outer(F00,My0,"*")+outer(F10,Hy0,"*")+outer(F01,My1,"*")+outer(F11,Hy1,"*") + + #Create output format + intsx <- lapply(as.list(1:nrow(spltCon)),function(x){ + matrix(rbind(spltx$ID[spltCon[x,]],cbind(fx[,x],fy[,x])),ncol=2, + dimnames=list(c("startendVMS",seq(1,res,1)),c("x","y")))}) + return(intsx)} + +rbindTacsat <- function(set1,set2){ + cln1 <- colnames(set1) + cln2 <- colnames(set2) + if(any(duplicated(cln1)==TRUE) || any(duplicated(cln2)==TRUE)) stop("Duplicate column names in datasets") + idx1 <- which(is.na(pmatch(cln1,cln2))==TRUE) + idx2 <- which(is.na(pmatch(cln2,cln1))==TRUE) + + if(length(idx1)>0){ + for(i in idx1) set2 <- cbind(set2,NA) + colnames(set2) <- c(cln2,cln1[idx1])} + if(length(idx2)>0){ + for(i in idx2) set1 <- cbind(set1,NA) + colnames(set1) <- c(cln1,cln2[idx2])} + cln1 <- colnames(set1) + cln2 <- colnames(set2) + mtch <- pmatch(cln1,cln2) + if(any(is.na(mtch))==TRUE) stop("Cannot find nor create all matching column names") + set3 <- rbind(set1,set2[,cln2[mtch]]) + return(set3)} + +bearing <- function(lon,lat,lonRef,latRef){ + + x1 <- lon + y1 <- lat + x2 <- lonRef + y2 <- latRef + + y <- sin((x2-x1)*pi/180) * cos(y2*pi/180) + x <- cos(y1*pi/180) * sin(y2*pi/180) - sin(y1*pi/180) * cos(y2*pi/180) * cos((x2-x1)*pi/180) + bearing <- atan2(y,x)*180/pi + bearing <- (bearing + 360)%%360 + return(bearing)} + +`distance` <- + function(lon,lat,lonRef,latRef){ + + pd <- pi/180 + + a1<- sin(((latRef-lat)*pd)/2) + a2<- cos(lat*pd) + a3<- cos(latRef*pd) + a4<- sin(((lonRef-lon)*pd)/2) + a <- a1*a1+a2*a3*a4*a4 + + c <- 2*atan2(sqrt(a),sqrt(1-a)); + return(6371*c)} + +distanceInterpolation <- function(interpolation){ + + res <- unlist(lapply(interpolation,function(x){ + dims <- dim(x) + res <- distance(x[3:dims[1],1],x[3:dims[1],2],x[2:(dims[1]-1),1],x[2:(dims[1]-1),2]) + return(sum(res,na.rm=TRUE))})) + + return(res)} + + +equalDistance <- function(interpolation,res=10){ + + #Calculate ditance of all interpolations at the same time + totDist <- distanceInterpolation(interpolation) + #Get dimensions of interpolations + lngInt <- lapply(interpolation,dim) + + #Warn if resolution of equal distance is too high compared to original resolution of interpolation + if(min(unlist(lngInt)[seq(1,length(totDist),2)],na.rm=TRUE) < 9*res) warnings("Number of intermediate points in the interpolation might be too small for the equal distance pionts chosen") + + #Get distance steps to get equal distance + eqStep <- totDist/(res-1) + + #Get x-y values of all interpolations + intidx <- matrix(unlist(lapply(interpolation,function(x){return(x[1,])})),ncol=2,byrow=TRUE) + + #Do the calculation + result <- lapply(interpolation,function(ind){ + i <- which(intidx[,1] == ind[1,1] & intidx[,2] == ind[1,2]) + idx <- apply(abs(outer( + cumsum(distance(ind[3:lngInt[[i]][1],1],ind[3:lngInt[[i]][1],2],ind[2:(lngInt[[i]][1]-1),1],ind[2:(lngInt[[i]][1]-1),2])), + seq(eqStep[i],totDist[i],eqStep[i]), + "-")), + 2,which.min)+1 + idx <- c(1,idx) + return(ind[c(1,idx+1),])}) + #Return the equal distance interpolated set in the same format as the interpolated dataset (as a list) + return(result)} + +interStraightLine <- function(spltx,spltCon,res){ + + fx <- mapply(seq,spltx[spltCon[,1],"SI_LONG"],spltx[spltCon[,2],"SI_LONG"],length.out=res) + fy <- mapply(seq,spltx[spltCon[,1],"SI_LATI"],spltx[spltCon[,2],"SI_LATI"],length.out=res) + + #Create output format + intsx <- lapply(as.list(1:nrow(spltCon)),function(x){ + matrix(rbind(spltx$ID[spltCon[x,]],cbind(fx[,x],fy[,x])),ncol=2, + dimnames=list(c("startendVMS",seq(1,res,1)),c("x","y")))}) + return(intsx)} + +interpolation2Tacsat <- function(interpolation,tacsat,npoints=10,equalDist=TRUE){ + + # This function takes the list of tracks output by interpolateTacsat and converts them back to tacsat format. + # The npoints argument is the optional number of points between each 'real' position. + tacsat <- sortTacsat(tacsat) + if(!"HL_ID" %in% colnames(tacsat)) tacsat$HL_ID <- 1:nrow(tacsat) + if(!"SI_DATIM" %in% colnames(tacsat)) tacsat$SI_DATIM <- as.POSIXct(paste(tacsat$SI_DATE, tacsat$SI_TIME, sep=" "), tz="GMT", format="%d/%m/%Y %H:%M") + if(equalDist){ + interpolationEQ <- equalDistance(interpolation,npoints) #Divide points equally along interpolated track (default is 10). + } else { + interpolationEQ <- lapply(interpolation,function(x){idx <- round(seq(2,nrow(x),length.out=npoints)); return(x[c(1,idx),])}) + } + res <- lapply(interpolationEQ,function(x){ + idx <- unlist(x[1,1:2]@.Data); x <- data.frame(x) + colnames(x) <- c("SI_LONG","SI_LATI") + cls <- which(apply(tacsat[c(idx),],2,function(y){return(length(unique(y)))})==1) + for(i in cls){ + x <- cbind(x,rep(tacsat[idx[1],i],nrow(x))); + colnames(x) <- c(colnames(x)[1:(ncol(x)-1)],colnames(tacsat)[i]) + } + if(!"VE_COU" %in% colnames(x)) x$VE_COU <- rep(tacsat$VE_COU[idx[1]],nrow(x)) + if(!"VE_REF" %in% colnames(x)) x$VE_REF <- rep(tacsat$VE_REF[idx[1]],nrow(x)) + if(!"FT_REF" %in% colnames(x)) x$FT_REF <- rep(tacsat$FT_REF[idx[1]],nrow(x)) + x$SI_DATIM <- tacsat$SI_DATIM[idx[1]] + x$SI_DATIM[-c(1:2)] <- as.POSIXct(cumsum(rep(difftime(tacsat$SI_DATIM[idx[2]],tacsat$SI_DATIM[idx[1]],units="secs")/(nrow(x)-2),nrow(x)-2))+tacsat$SI_DATIM[idx[1]],tz="GMT",format = "%d/%m/%Y %H:%M") + x$SI_DATE <- format(x$SI_DATIM,format="%d/%m/%Y") + timeNotation <- ifelse(length(unlist(strsplit(tacsat$SI_TIME[1],":")))>2,"secs","mins") + if(timeNotation == "secs") x$SI_TIME <- format(x$SI_DATIM,format="%H:%M:%S") + if(timeNotation == "mins") x$SI_TIME <- format(x$SI_DATIM,format="%H:%M") + x$SI_SP <- mean(c(tacsat$SI_SP[idx[1]],tacsat$SI_SP[idx[2]]),na.rm=TRUE) + x$SI_HE <- NA; + x$SI_HE[-c(1,nrow(x))] <- bearing(x$SI_LONG[2:(nrow(x)-1)],x$SI_LATI[2:(nrow(x)-1)],x$SI_LONG[3:nrow(x)],x$SI_LATI[3:nrow(x)]) + x$HL_ID <- tacsat$HL_ID[idx[1]] + return(x[-c(1,2,nrow(x)),])}) + + #interpolationTot <- do.call(rbind,res) + interpolationTot <- res[[1]][,which(duplicated(colnames(res[[1]]))==FALSE)] + if(length(res)>1){ + for(i in 2:length(res)){ + if(nrow(res[[i]])>0) + interpolationTot <- rbindTacsat(interpolationTot,res[[i]][,which(duplicated(colnames(res[[i]]))==FALSE)]) + } + } + #tacsatInt <- rbind(interpolationTot,tacsat[,colnames(interpolationTot)]) + tacsatInt <- rbindTacsat(tacsat,interpolationTot) + tacsatInt <- sortTacsat(tacsatInt) + + return(tacsatInt) + +} + +`sortTacsat` <- + function(dat){ + require(doBy) + + if(!"SI_DATIM" %in% colnames(dat)) dat$SI_DATIM <- as.POSIXct(paste(dat$SI_DATE, dat$SI_TIME, sep=" "), tz="GMT", format="%d/%m/%Y %H:%M") + + #Sort the tacsat data first by ship, then by date + if("VE_REF" %in% colnames(dat)) dat <- orderBy(~VE_REF+SI_DATIM,data=dat) + if("OB_REF" %in% colnames(dat)) dat <- orderBy(~OB_REF+SI_DATIM,data=dat) + + return(dat)} + +`lonLatRatio` <- + function(x1,lat){ + #Based on the Haversine formula + #At the position, the y-position remains the same, hence, cos(lat)*cos(lat) instead of cos(lat) * cos(y2) + a <- cos(lat*pi/180)*cos(lat*pi/180)*sin((0.1*pi/180)/2)*sin((0.1*pi/180)/2); + c <- 2*atan2(sqrt(a),sqrt(1-a)); + R <- 6371; + dx1 <- R*c + + return(c(dx1/11.12))} + + +`an` <- + function(x){return(as.numeric(x))} + + +`findEndTacsat` <- + function(tacsat + ,startTacsat #Starting point of VMS + ,interval #Specify in minutes, NULL means use all points + ,margin #Specify the margin in minutes it might deviate from the interval time, in minutes + ){ + VMS <- tacsat + if(!"SI_DATIM" %in% colnames(VMS)) VMS$SI_DATIM <- as.POSIXct(paste(tacsat$SI_DATE, tacsat$SI_TIME, sep=" "), tz="GMT", format="%d/%m/%Y %H:%M") + + startVMS <- startTacsat + clStartVMS <- startVMS #Total VMS list starting point instead of subset use + iShip <- VMS$VE_REF[startVMS] + VMS. <- subset(VMS,VE_REF==iShip) + startVMS <- which(VMS$VE_REF[startVMS] == VMS.$VE_REF & VMS$SI_DATIM[startVMS] == VMS.$SI_DATIM) + if(clStartVMS != dim(VMS)[1]){ + if(VMS$VE_REF[clStartVMS] != VMS$VE_REF[clStartVMS+1]){ + #End of dataset reached + endDataSet <- 1 + endVMS <- NA + } else { + #Calculate the difference in time between the starting VMS point and its succeeding points + diffTime <- difftime(VMS.$SI_DATIM[(startVMS+1):dim(VMS.)[1]],VMS.$SI_DATIM[startVMS],units=c("mins")) + if(length(which(diffTime >= (interval-margin) & diffTime <= (interval+margin)))==0){ + warning("No succeeding point found, no interpolation possible") + endVMS <- NA + #Check if end of dataset has been reached + ifelse(all((diffTime < (interval-margin))==TRUE),endDataSet <- 1,endDataSet <- 0) + } else { + res <- which(diffTime >= (interval-margin) & diffTime <= (interval+margin)) + if(length(res)>1){ + res2 <- which.min(abs(interval-an(diffTime[res]))) + endVMS <- startVMS + res[res2] + endDataSet <- 0 + } else { + endVMS <- startVMS + res + endDataSet <- 0 + } + } + #Build-in check + if(is.na(endVMS)==FALSE){ + if(!an(difftime(VMS.$SI_DATIM[endVMS],VMS.$SI_DATIM[startVMS],units=c("mins"))) %in% seq((interval-margin),(interval+margin),1)) stop("found endVMS point not within interval range") + endVMS <- clStartVMS + (endVMS - startVMS) + } + + } + } else { endDataSet <- 1; endVMS <- NA} + + return(c(endVMS,endDataSet))} + +`interpolateTacsat` <- +function(tacsat #VMS datapoints + ,interval=120 #Specify in minutes, NULL means use all points + ,margin=12 #Specify the margin in minutes that the interval might deviate in a search for the next point + ,res=100 #Resolution of interpolation method (default = 100) + ,method="cHs" #Specify the method to be used: Straight line (SL) of cubic Hermite spline (cHs) + ,params=list(fm=0.5,distscale=20,sigline=0.2,st=c(2,6)) #Specify the three parameters: fm, distscale, sigline, speedthreshold + ,headingAdjustment=0 + ,fast=FALSE){ + +if(!"SI_DATIM" %in% colnames(tacsat)) tacsat$SI_DATIM <- as.POSIXct(paste(tacsat$SI_DATE, tacsat$SI_TIME, sep=" "), tz="GMT", format="%d/%m/%Y %H:%M") + + #Start interpolating the data +if(!method %in% c("cHs","SL")) stop("method selected that does not exist") + +#------------------------------------------------------------------------------- +#Fast method or not +#------------------------------------------------------------------------------- +if(fast){ + #Interpolation only by vessel, so split tacsat up + tacsat$ID <- 1:nrow(tacsat) + splitTa <- split(tacsat,tacsat$VE_REF) + spltTaCon <- lapply(splitTa,function(spltx){ + #Calculate time different between every record + dftimex <- outer(spltx$SI_DATIM,spltx$SI_DATIM,difftime,units="mins") + iStep <- 1 + connect <- list() + counter <- 1 + #Loop over all possible combinations and store if a connection can be made + while(iStep <= nrow(spltx)){ + endp <- which(dftimex[,iStep] >= (interval - margin) & dftimex[,iStep] <= (interval + margin)) + if(length(endp)>0){ + if(length(endp)>1) endp <- endp[which.min(abs(interval - dftimex[endp,iStep]))][1] + connect[[counter]] <- c(iStep,endp) + counter <- counter + 1 + iStep <- endp + } else { iStep <- iStep + 1} + } + #Return matrix of conenctions + return(do.call(rbind,connect))}) + + if(method=="cHs") returnInterpolations <- unlist(lapply(as.list(names(unlist(lapply(spltTaCon,nrow)))),function(y){ + return(interCubicHermiteSpline(spltx=splitTa[[y]],spltCon=spltTaCon[[y]],res,params,headingAdjustment))}),recursive=FALSE) + if(method=="SL") returnInterpolations <- unlist(lapply(as.list(names(unlist(lapply(spltTaCon,nrow)))),function(y){ + return(interStraightLine(splitTa[[y]],spltTaCon[[y]],res))}),recursive=FALSE) + +} else { + + + #Initiate returning result object + returnInterpolations <- list() + + #Start iterating over succeeding points + for(iStep in 1:(dim(tacsat)[1]-1)){ + if(iStep == 1){ + iSuccess <- 0 + endDataSet <- 0 + startVMS <- 1 + ship <- tacsat$VE_REF[startVMS] + } else { + if(is.na(endVMS)==TRUE) endVMS <- startVMS + 1 + startVMS <- endVMS + #-Check if the end of the dataset is reached + if(endDataSet == 1 & rev(unique(tacsat$VE_REF))[1] != ship){ + startVMS <- which(tacsat$VE_REF == unique(tacsat$VE_REF)[which(unique(tacsat$VE_REF)==ship)+1])[1] + ship <- tacsat$VE_REF[startVMS] + endDataSet<- 0 + } + if(endDataSet == 1 & rev(unique(tacsat$VE_REF))[1] == ship) endDataSet <- 2 #Final end of dataset + } + + #if end of dataset is not reached, try to find succeeding point + if(endDataSet != 2){ + result <- findEndTacsat(tacsat,startVMS,interval,margin) + endVMS <- result[1] + endDataSet <- result[2] + if(is.na(endVMS)==TRUE) int <- 0 #No interpolation possible + if(is.na(endVMS)==FALSE) int <- 1 #Interpolation possible + + #Interpolate according to the Cubic Hermite Spline method + if(method == "cHs" & int == 1){ + + #Define the cHs formula + F00 <- numeric() + F10 <- numeric() + F01 <- numeric() + F11 <- numeric() + i <- 0 + t <- seq(0,1,length.out=res) + F00 <- 2*t^3 -3*t^2 + 1 + F10 <- t^3-2*t^2+t + F01 <- -2*t^3+3*t^2 + F11 <- t^3-t^2 + + if (is.na(tacsat[startVMS,"SI_HE"])=="TRUE") tacsat[startVMS,"SI_HE"] <- 0 + if (is.na(tacsat[endVMS, "SI_HE"])=="TRUE") tacsat[endVMS, "SI_HE"] <- 0 + + #Heading at begin point in degrees + Hx0 <- sin(tacsat[startVMS,"SI_HE"]/(180/pi)) + Hy0 <- cos(tacsat[startVMS,"SI_HE"]/(180/pi)) + #Heading at end point in degrees + Hx1 <- sin(tacsat[endVMS-headingAdjustment,"SI_HE"]/(180/pi)) + Hy1 <- cos(tacsat[endVMS-headingAdjustment,"SI_HE"]/(180/pi)) + + Mx0 <- tacsat[startVMS, "SI_LONG"] + Mx1 <- tacsat[endVMS, "SI_LONG"] + My0 <- tacsat[startVMS, "SI_LATI"] + My1 <- tacsat[endVMS, "SI_LATI"] + + #Corrected for longitude lattitude effect + Hx0 <- Hx0 * params$fm * tacsat[startVMS,"SI_SP"] /((params$st[2]-params$st[1])/2+params$st[1]) + Hx1 <- Hx1 * params$fm * tacsat[endVMS,"SI_SP"] /((params$st[2]-params$st[1])/2+params$st[1]) + Hy0 <- Hy0 * params$fm * lonLatRatio(tacsat[c(startVMS,endVMS),"SI_LONG"],tacsat[c(startVMS,endVMS),"SI_LATI"])[1] * tacsat[startVMS,"SI_SP"]/((params$st[2]-params$st[1])/2+params$st[1]) + Hy1 <- Hy1 * params$fm * lonLatRatio(tacsat[c(startVMS,endVMS),"SI_LONG"],tacsat[c(startVMS,endVMS),"SI_LATI"])[2] * tacsat[endVMS,"SI_SP"]/((params$st[2]-params$st[1]) /2+params$st[1]) + + #Finalizing the interpolation based on cHs + fx <- numeric() + fy <- numeric() + fx <- F00*Mx0+F10*Hx0+F01*Mx1+F11*Hx1 + fy <- F00*My0+F10*Hy0+F01*My1+F11*Hy1 + + #Add one to list of successful interpolations + iSuccess <- iSuccess + 1 + returnInterpolations[[iSuccess]] <- matrix(rbind(c(startVMS,endVMS),cbind(fx,fy)),ncol=2,dimnames=list(c("startendVMS",seq(1,res,1)),c("x","y"))) + } + + #Interpolate according to a straight line + if(method == "SL" & int == 1){ + fx <- seq(tacsat$SI_LONG[startVMS],tacsat$SI_LONG[endVMS],length.out=res) + fy <- seq(tacsat$SI_LATI[startVMS],tacsat$SI_LATI[endVMS],length.out=res) + + #Add one to list of successful interpolations + iSuccess <- iSuccess + 1 + returnInterpolations[[iSuccess]] <- matrix(rbind(c(startVMS,endVMS),cbind(fx,fy)),ncol=2,dimnames=list(c("startendVMS",seq(1,res,1)),c("x","y"))) + } + } + } +} + +return(returnInterpolations)} + + +cat("Loading Table\n") +tacsatX <-read.table(inputFile,sep=",",header=T) +cat("Adjusting Columns Types\n") +tacsatX<-transform(tacsatX, VE_COU= as.character(VE_COU), VE_REF= as.character(VE_REF), SI_LATI= as.numeric(SI_LATI), SI_LONG= as.numeric(SI_LONG), SI_DATE= as.character(SI_DATE),SI_TIME= as.character(SI_TIME),SI_SP= as.numeric(SI_SP),SI_HE= as.numeric(SI_HE)) +tacsatX$SI_DATIM=NULL +cat("Sorting dataset\n") +tacsatS <- sortTacsat(tacsatX) +tacsatCut<-tacsatS +tacsatCut <- tacsatS[1:1000,] + +cat("Interpolating\n") +interpolation <- interpolateTacsat(tacsatCut,interval=interval,margin=margin,res=res, method=method,params=list(fm=fm,distscale=distscale,sigline=sigline,st=st),headingAdjustment=headingAdjustment,fast=fast) +cat("Reconstructing Dataset\n") +tacsatInt <- interpolation2Tacsat(interpolation=interpolation,tacsat=tacsatCut,npoints=npoints,equalDist=equalDist) +tacsatInt <- sortTacsat(tacsatInt) +cat("Writing output file\n") +write.csv(tacsatInt, outputFile, row.names=T) +print(Sys.time()) +cat("All Done.\n") diff --git a/cfg/modelers.properties b/cfg/modelers.properties new file mode 100644 index 0000000..d5c652e --- /dev/null +++ b/cfg/modelers.properties @@ -0,0 +1 @@ +HSPEN_MODELER=org.gcube.dataanalysis.ecoengine.modeling.SimpleModeler \ No newline at end of file diff --git a/cfg/models.properties b/cfg/models.properties new file mode 100644 index 0000000..479b406 --- /dev/null +++ b/cfg/models.properties @@ -0,0 +1,4 @@ +HSPEN=org.gcube.dataanalysis.ecoengine.models.ModelHSPEN +AQUAMAPSNN=org.gcube.dataanalysis.ecoengine.models.ModelAquamapsNN +FEED_FORWARD_ANN=org.gcube.dataanalysis.ecoengine.models.FeedForwardNN +FEED_FORWARD_ANN_FILE=org.gcube.dataanalysis.ecoengine.models.testing.FeedForwardNNFile \ No newline at end of file diff --git a/cfg/nodealgorithms.properties b/cfg/nodealgorithms.properties new file mode 100644 index 0000000..b36fa1d --- /dev/null +++ b/cfg/nodealgorithms.properties @@ -0,0 +1,11 @@ +AQUAMAPS_SUITABLE=org.gcube.dataanalysis.executor.nodes.algorithms.AquamapsSuitableNode +AQUAMAPS_NATIVE=org.gcube.dataanalysis.executor.nodes.algorithms.AquamapsNativeNode +AQUAMAPS_NATIVE_2050=org.gcube.dataanalysis.executor.nodes.algorithms.AquamapsNative2050Node +AQUAMAPS_SUITABLE_2050=org.gcube.dataanalysis.executor.nodes.algorithms.AquamapsSuitable2050Node +OCCURRENCES_MERGER=org.gcube.dataanalysis.executor.nodes.transducers.OccurrenceMergingNode +OCCURRENCES_INTERSECTOR=org.gcube.dataanalysis.executor.nodes.transducers.OccurrenceIntersectionNode +OCCURRENCES_SUBTRACTION=org.gcube.dataanalysis.executor.nodes.transducers.OccurrenceSubtractionNode +LWR=org.gcube.dataanalysis.executor.nodes.algorithms.LWR +BIONYM=org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer +BIONYM_BIODIV=org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymBiodiv +CMSY=org.gcube.dataanalysis.executor.nodes.algorithms.CMSY \ No newline at end of file diff --git a/cfg/operators.xml b/cfg/operators.xml new file mode 100644 index 0000000..93b465c --- /dev/null +++ b/cfg/operators.xml @@ -0,0 +1,2620 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cfg/transducerers.properties b/cfg/transducerers.properties new file mode 100644 index 0000000..5aaa938 --- /dev/null +++ b/cfg/transducerers.properties @@ -0,0 +1,16 @@ +BIOCLIMATE_HSPEC=org.gcube.dataanalysis.ecoengine.transducers.BioClimateHSPECTransducer +BIOCLIMATE_HCAF=org.gcube.dataanalysis.ecoengine.transducers.BioClimateHCAFTransducer +BIOCLIMATE_HSPEN=org.gcube.dataanalysis.ecoengine.transducers.BioClimateHSPENTransducer +HCAF_INTERPOLATION=org.gcube.dataanalysis.ecoengine.transducers.InterpolationTransducer +HCAF_FILTER=org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors.HcafFilter +HSPEN_FILTER=org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors.HspenFilter +ABSENCE_CELLS_FROM_AQUAMAPS=org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors.MarineAbsencePointsFromAquamapsDistribution +PRESENCE_CELLS_GENERATION=org.gcube.dataanalysis.ecoengine.transducers.simplequeryexecutors.MarinePresencePoints +OCCURRENCES_MERGER=org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsMerger +OCCURRENCES_INTERSECTOR=org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsIntersector +OCCURRENCES_MARINE_TERRESTRIAL=org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsInSeaOnEarth +OCCURRENCES_DUPLICATES_DELETER=org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsDuplicatesDeleter +OCCURRENCES_SUBTRACTION=org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsSubtraction +BIONYM=org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymTransducer +BIONYM_LOCAL=org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer +SGVM_INTERPOLATION=org.gcube.dataanalysis.executor.rscripts.SGVMS_Interpolation \ No newline at end of file diff --git a/cfg/userperspective.properties b/cfg/userperspective.properties new file mode 100644 index 0000000..66fc553 --- /dev/null +++ b/cfg/userperspective.properties @@ -0,0 +1,13 @@ +ANOMALIES_DETECTION=DBSCAN,KMEANS,XMEANS +CLASSIFICATION=FEED_FORWARD_A_N_N_DISTRIBUTION +CLIMATE=BIOCLIMATE_HSPEC,BIOCLIMATE_HCAF,BIOCLIMATE_HSPEN,HCAF_INTERPOLATION +CORRELATION_ANALYSIS=HRS +DATA_CLUSTERING=DBSCAN,KMEANS,XMEANS +FILTERING=HCAF_FILTER,HSPEN_FILTER +FUNCTION_SIMULATION=FEED_FORWARD_A_N_N_DISTRIBUTION +OCCURRENCES=ABSENCE_CELLS_FROM_AQUAMAPS,PRESENCE_CELLS_GENERATION,OCCURRENCES_MERGER,OCCURRENCES_INTERSECTOR,OCCURRENCES_MARINE_TERRESTRIAL,OCCURRENCES_DUPLICATES_DELETER,OCCURRENCES_SUBTRACTION +PERFORMANCES_EVALUATION=QUALITY_ANALYSIS,DISCREPANCY_ANALYSIS +SPECIES_SIMULATION=AQUAMAPS_SUITABLE,AQUAMAPS_NATIVE,AQUAMAPS_NATIVE_2050,AQUAMAPS_SUITABLE_2050,AQUAMAPS_NATIVE_NEURALNETWORK,AQUAMAPS_SUITABLE_NEURALNETWORK +TRAINING=HSPEN,AQUAMAPSNN,FEED_FORWARD_ANN +TIME_SERIES=HCAF_INTERPOLATION +VESSELS=SGVM_INTERPOLATION \ No newline at end of file diff --git a/createscript.jardesc b/createscript.jardesc new file mode 100644 index 0000000..b01ebc3 --- /dev/null +++ b/createscript.jardesc @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/deployDesktop.jardesc b/deployDesktop.jardesc new file mode 100644 index 0000000..7595d24 --- /dev/null +++ b/deployDesktop.jardesc @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/deployParallelProcessing.jardesc b/deployParallelProcessing.jardesc new file mode 100644 index 0000000..f9020ad --- /dev/null +++ b/deployParallelProcessing.jardesc @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/distro/CHANGELOG b/distro/CHANGELOG new file mode 100644 index 0000000..5ff0fb3 --- /dev/null +++ b/distro/CHANGELOG @@ -0,0 +1,2 @@ +v. 1.0.0 (20-04-2011) + * First release diff --git a/distro/INSTALL b/distro/INSTALL new file mode 100644 index 0000000..9bcbb15 --- /dev/null +++ b/distro/INSTALL @@ -0,0 +1 @@ +Used as a library in the gCube Framework \ No newline at end of file diff --git a/distro/LICENSE b/distro/LICENSE new file mode 100644 index 0000000..bd0c2b4 --- /dev/null +++ b/distro/LICENSE @@ -0,0 +1,8 @@ +gCube System - License +------------------------------------------------------------ + +The gCube/gCore software is licensed as Free Open Source software conveying to +the EUPL (http://ec.europa.eu/idabc/eupl). +The software and documentation is provided by its authors/distributors "as is" +and no expressed or implied warranty is given for its use, quality or fitness +for a particular case. diff --git a/distro/MAINTAINERS b/distro/MAINTAINERS new file mode 100644 index 0000000..1f1e62d --- /dev/null +++ b/distro/MAINTAINERS @@ -0,0 +1,2 @@ +Gianpaolo Coro (gianpaolo.coro@isti.cnr.it), CNR Pisa, +Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" \ No newline at end of file diff --git a/distro/README b/distro/README new file mode 100644 index 0000000..f0a5e69 --- /dev/null +++ b/distro/README @@ -0,0 +1,42 @@ +The gCube System - Ecological Engine Library +------------------------------------------------------------ + +This work is partially funded by the European Commission in the +context of the D4Science project (www.d4science.eu), under the +1st call of FP7 IST priority. + +Authors +------- + +* Gianpaolo Coro (gianpaolo.coro@isti.cnr.it), CNR Pisa, + Istituto di Scienza e Tecnologie dell'Informazione "A. Faedo" + +Version and Release Date +------------------------ + +version 1.0.0 (22-06-2012) + +Description +-------------------- + +Support library for statistics analysis on Time Series data. + + +Download information +-------------------- + +Source code is available from SVN: +http://svn.research-infrastructures.eu/d4science/gcube/trunk/content-management/EcologicalModelling + +Binaries can be downloaded from: +http://software.d4science.research-infrastructures.eu/ + +Documentation +------------- +VREManager documentation is available on-line from the Projects Documentation Wiki: +https://gcube.wiki.gcube-system.org/gcube/index.php/Ecological_Modeling + +Licensing +--------- + +This software is licensed under the terms you may find in the file named "LICENSE" in this directory. diff --git a/distro/changelog.xml b/distro/changelog.xml new file mode 100644 index 0000000..902ffc5 --- /dev/null +++ b/distro/changelog.xml @@ -0,0 +1,7 @@ + + + First Release + + \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml new file mode 100644 index 0000000..21d8c88 --- /dev/null +++ b/distro/descriptor.xml @@ -0,0 +1,42 @@ + + servicearchive + + tar.gz + + / + + + ${distroDirectory} + / + true + + README + LICENSE + INSTALL + MAINTAINERS + changelog.xml + + 755 + true + + + + + ${distroDirectory}/profile.xml + / + true + + + target/${build.finalName}.jar + /${artifactId} + + + ${distroDirectory}/svnpath.txt + /${artifactId} + true + + + \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml new file mode 100644 index 0000000..a8760e3 --- /dev/null +++ b/distro/profile.xml @@ -0,0 +1,25 @@ + + + + Library + + Ecological Engine Executor Library + EcologicalEngineExecutor + ${artifactId} + 1.0.0 + + + ${artifactId} + ${version} + + ${groupId} + ${artifactId} + ${version} + + + ${build.finalName}.jar + + + + + \ No newline at end of file diff --git a/distro/svnpath.txt b/distro/svnpath.txt new file mode 100644 index 0000000..3f85e8c --- /dev/null +++ b/distro/svnpath.txt @@ -0,0 +1 @@ +https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngine \ No newline at end of file diff --git a/etc/executorscript.profile b/etc/executorscript.profile new file mode 100644 index 0000000..64980fb --- /dev/null +++ b/etc/executorscript.profile @@ -0,0 +1,28 @@ + + + + Service + + A Plugin Executing S.O. Scripts + ExecutorPlugins + ExecutorScript + 1.0.0 + + + plugin + 1.0.0 + + + VREManagement + Executor + 1.1.0 + + main + 1.0.0 + + org.gcube.dataanalysis.executor.plugin.ScriptPluginContext + org.gcube.dataanalysis.executor.executorscriptplugin.jar + + + + \ No newline at end of file diff --git a/etc/profile.xml b/etc/profile.xml new file mode 100644 index 0000000..64980fb --- /dev/null +++ b/etc/profile.xml @@ -0,0 +1,28 @@ + + + + Service + + A Plugin Executing S.O. Scripts + ExecutorPlugins + ExecutorScript + 1.0.0 + + + plugin + 1.0.0 + + + VREManagement + Executor + 1.1.0 + + main + 1.0.0 + + org.gcube.dataanalysis.executor.plugin.ScriptPluginContext + org.gcube.dataanalysis.executor.executorscriptplugin.jar + + + + \ No newline at end of file diff --git a/neuralnetwork_Fis-30189_staticsuitable_neuralname b/neuralnetwork_Fis-30189_staticsuitable_neuralname new file mode 100644 index 0000000..3456fee Binary files /dev/null and b/neuralnetwork_Fis-30189_staticsuitable_neuralname differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..3046f21 --- /dev/null +++ b/pom.xml @@ -0,0 +1,138 @@ + + 4.0.0 + + maven-parent + org.gcube.tools + 1.0.0 + + + org.gcube.dataanalysis + EcologicalEngineExecutor + 1.6.4-SNAPSHOT + ecological-engine-executor + ecological-engine-executor + + ${project.basedir}/distro + + + + org.gcube.resourcemanagement + executor-service + [1.2.0-SNAPSHOT,2.0.0-SNAPSHOT) + + + org.gcube.contentmanagement + storage-manager-core + [2.0.2-SNAPSHOT,4.0.0) + + + org.gcube.contentmanagement + storage-manager-wrapper + [2.0.2-SNAPSHOT,4.0.0) + + + org.apache.activemq + activemq-core + 5.6.0 + + + org.gcube.core + gcf + [1.4.1,2.0.0) + + + org.gcube.dataanalysis + ecological-engine + [1.8.0-SNAPSHOT,2.0.0) + + + org.gcube.informationsystem + is-client + [1.5.1,1.6.0] + + + org.gcube.informationsystem + is-collector-stubs + [3.0.0-SNAPSHOT, 3.1.0) + + + org.gcube.core + common-scope + [1.2.0-SNAPSHOT,3.0.0) + + + + + + + + maven-compiler-plugin + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12 + + true + + + + org.apache.maven.plugins + maven-resources-plugin + 2.5 + + + copy-profile + install + + copy-resources + + + target + + + ${distroDirectory} + true + + profile.xml + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.2 + + + ${distroDirectory}/descriptor.xml + + + + + servicearchive + install + + single + + + + + + + \ No newline at end of file diff --git a/src/main/java/org/gcube/dataanalysis/executor/generators/D4ScienceDistributedProcessing.java b/src/main/java/org/gcube/dataanalysis/executor/generators/D4ScienceDistributedProcessing.java new file mode 100644 index 0000000..1569599 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/generators/D4ScienceDistributedProcessing.java @@ -0,0 +1,148 @@ +package org.gcube.dataanalysis.executor.generators; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; +import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.interfaces.GenericAlgorithm; +import org.gcube.dataanalysis.executor.job.management.DistributedProcessingAgent; + +public class D4ScienceDistributedProcessing implements Generator { + public static int maxMessagesAllowedPerJob = 20; + public static boolean forceUpload = true; + public static String defaultContainerFolder = "PARALLEL_PROCESSING"; + protected AlgorithmConfiguration config; + protected ActorNode distributedModel; + protected String mainclass; + DistributedProcessingAgent agent; + + public D4ScienceDistributedProcessing(){ + } + + public D4ScienceDistributedProcessing(AlgorithmConfiguration config) { + this.config = config; + + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + } + + public void compute() throws Exception { + try { + agent.compute(); + distributedModel.postProcess(agent.hasResentMessages(),false); + } catch (Exception e) { + distributedModel.postProcess(false,true); + AnalysisLogger.getLogger().error("ERROR: An Error occurred ", e); + throw e; + } finally { + shutdown(); + } + } + + @Override + public List getInputParameters() { + + List distributionModelParams = new ArrayList(); + distributionModelParams.add(new ServiceType(ServiceParameters.USERNAME,"ServiceUserName","The final user Name")); + + return distributionModelParams; + } + + + @Override + public String getResources() { + return agent.getResources(); + } + + @Override + public float getStatus() { + return agent.getStatus(); + } + + @Override + public StatisticalType getOutput() { + return distributedModel.getOutput(); + } + + @Override + public ALG_PROPS[] getSupportedAlgorithms() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON}; + return p; + } + + @Override + public INFRASTRUCTURE getInfrastructure() { + return INFRASTRUCTURE.D4SCIENCE; + } + + @Override + public void init() throws Exception { + + Properties p = AlgorithmConfiguration.getProperties(config.getConfigPath() + AlgorithmConfiguration.nodeAlgorithmsFile); + String model = config.getModel(); + String algorithm = null; + if ((model!=null) && (model.length()>0)) + algorithm = model; + else + algorithm=config.getAgent(); + + mainclass = p.getProperty(algorithm); + distributedModel = (ActorNode) Class.forName(mainclass).newInstance(); + distributedModel.setup(config); + String scope = config.getGcubeScope(); + AnalysisLogger.getLogger().info("Using the following scope for the computation:"+scope); + String owner = config.getParam("ServiceUserName"); + int leftNum = distributedModel.getNumberOfLeftElements(); + int rightNum = distributedModel.getNumberOfRightElements(); + agent = new DistributedProcessingAgent(config, scope, owner, mainclass, config.getPersistencePath(), algorithm, defaultContainerFolder, maxMessagesAllowedPerJob, forceUpload, leftNum, rightNum); + agent.setLogger(AnalysisLogger.getLogger()); + } + + @Override + public void setConfiguration(AlgorithmConfiguration config) { + this.config = config; + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + } + + @Override + public void shutdown() { + try { + agent.shutdown(); + } catch (Exception e) { + } + try { + distributedModel.stop(); + } catch (Exception e) { + } + } + + @Override + public String getLoad() { + return agent.getLoad(); + } + + @Override + public String getResourceLoad() { + return agent.getResourceLoad(); + } + + + @Override + public GenericAlgorithm getAlgorithm() { + return distributedModel; + } + + @Override + public String getDescription() { + return "A D4Science Cloud Processor for Species Distributions"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/job/management/DistributedProcessingAgent.java b/src/main/java/org/gcube/dataanalysis/executor/job/management/DistributedProcessingAgent.java new file mode 100644 index 0000000..25a8972 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/job/management/DistributedProcessingAgent.java @@ -0,0 +1,293 @@ +package org.gcube.dataanalysis.executor.job.management; + +import java.io.File; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.axis.message.addressing.Address; +import org.apache.axis.message.addressing.EndpointReferenceType; +import org.apache.log4j.Logger; +import org.gcube.contentmanagement.graphtools.utils.HttpRequest; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; +import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.ResourceLoad; +import org.gcube.dataanalysis.ecoengine.connectors.livemonitor.Resources; +import org.gcube.dataanalysis.ecoengine.utils.Operations; + +import com.thoughtworks.xstream.XStream; + +public class DistributedProcessingAgent { + + protected QueueJobManager jobManager; + protected boolean deletefiles = true; + protected String mainclass; + public int maxElementsAllowedPerJob = 20; + protected boolean forceUpload = true; + protected boolean stop; + protected String gscope; + protected String userName; + protected String pathToLib; + protected String modelName; + protected String containerFolder; + protected Serializable configurationFile; + protected int rightSetNumberOfElements; + protected int leftSetNumberOfElements; + protected List endpoints; + protected int subdivisiondiv; + + protected static String defaultJobOutput = "execution.output"; + protected static String defaultScriptFile = "script"; + protected Logger logger; + + /** + * A distributed processing agent. Performs a distributed computation doing the MAP of the product of two sets: A and B + * Splits over B : A x B1 , A x B2, ... , A x Bn + * Prepares a script to be executed on remote nodes + * The computation is then sent to remote processors. + */ + public DistributedProcessingAgent(Serializable configurationFile, + String gCubeScope, + String computationOwner, + String mainClass, + String pathToLibFolder, + String modelName, + String containerFolder, + int maxElementsPerJob, + boolean forceReUploadofLibs, + int leftSetNumberOfElements, + int rightSetNumberOfElements + ) { + this.stop = false; + this.deletefiles = true; + this.gscope=gCubeScope; + this.mainclass=mainClass; + this.maxElementsAllowedPerJob=maxElementsPerJob; + this.forceUpload=forceReUploadofLibs; + this.configurationFile=configurationFile; + this.rightSetNumberOfElements=rightSetNumberOfElements; + this.leftSetNumberOfElements=leftSetNumberOfElements; + this.userName=computationOwner; + this.pathToLib=pathToLibFolder; + this.modelName=modelName; + this.containerFolder=containerFolder; + } + + public void setLogger(Logger logger){ + this.logger=logger; + } + + public void setEndPoints(List endpoints){ + this.endpoints=endpoints; + } + + public boolean hasResentMessages(){ + return jobManager.hasResentMessages(); + } + + public void compute() throws Exception { + try { + if (logger == null){ + logger = AnalysisLogger.getLogger(); + } + if (gscope == null) + throw new Exception("Null Scope"); + AnalysisLogger.getLogger().debug("SCOPE: "+gscope); + if (endpoints != null) { + List eprtList = new ArrayList(); + for (String ep : endpoints) { + eprtList.add(new EndpointReferenceType(new Address(ep))); + } + jobManager = new QueueJobManager(gscope, endpoints.size(), eprtList); + } else + jobManager = new QueueJobManager(gscope, 1); + + int numberOfResources = jobManager.getNumberOfNodes(); + // we split along right dimension so if elements are less than nodes, we should reduce the number of nodes + if (numberOfResources > 0) { + // chunkize the number of species in order to lower the computational effort of the workers + subdivisiondiv = rightSetNumberOfElements / (numberOfResources * maxElementsAllowedPerJob); + int rest = rightSetNumberOfElements % (numberOfResources * maxElementsAllowedPerJob); + if (rest > 0) + subdivisiondiv++; + if (subdivisiondiv == 0) + subdivisiondiv = 1; + + executeWork(leftSetNumberOfElements, rightSetNumberOfElements, 0, subdivisiondiv, deletefiles, forceUpload); + + if (jobManager.wasAborted()) { + logger.debug("Warning: Job was aborted"); +// distributionModel.postProcess(false,true); + throw new Exception("Job System Error"); + } + else{ + //postprocess +// distributionModel.postProcess(jobManager.hasResentMessages(),false); + } + + } else { + logger.debug("Warning: No Workers available"); + throw new Exception("No Workers available"); + } + + } catch (Exception e) { + logger.error("ERROR: An Error occurred ", e); + e.printStackTrace(); + throw e; + } finally { + shutdown(); + } + } + + private void executeWork(int leftNum, int rightNum, int offset, int numberOfResources, boolean deletefiles, boolean forceUpload) throws Exception { + + String owner = userName; + + int[] chunkSizes = Operations.takeChunks(rightNum, numberOfResources); + List arguments = new ArrayList(); + // chunkize respect to the cells: take a chunk of cells vs all species at each node! + + for (int i = 0; i < chunkSizes.length; i++) { + String argumentString = "0 " + leftNum + " " + offset + " " + chunkSizes[i] + " ./ "+mainclass; + arguments.add(argumentString); + offset += chunkSizes[i]; + logger.debug("Generator-> Argument " + i + ": " + argumentString); + } + + if (owner == null) + throw new Exception("Null Owner"); + + String pathToDir = new File (pathToLib, containerFolder).getAbsolutePath(); + + if (!(new File(pathToDir).exists())) + throw new Exception("No Implementation of node-model found for algorithm " + pathToDir); + + if (mainclass == null) + throw new Exception("No mainClass found for algorithm " + pathToDir); + + buildScriptFile(modelName, defaultJobOutput, pathToDir, mainclass); + + jobManager.uploadAndExecuteChunkized(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, owner, pathToDir, "/" + modelName + "/", "./", getScriptName(mainclass), arguments, new XStream().toXML(configurationFile), deletefiles, forceUpload); + + } + + private String getScriptName(String fullMainClass){ + String scriptName = defaultScriptFile+"_"+fullMainClass.substring(fullMainClass.lastIndexOf(".")+1)+".sh"; + return scriptName; + } + // builds a job.sh + public void buildScriptFile(String jobName, String jobOutput, String jarsPath, String fullMainClass) throws Exception { + File expectedscript = new File(jarsPath,getScriptName(fullMainClass)); + if (!expectedscript.exists()) { + StringBuffer sb = new StringBuffer(); + sb.append("#!/bin/sh\n"); + sb.append("# " + jobName + "\n"); + sb.append("cd $1\n"); + sb.append("\n"); + sb.append("java -Xmx1024M -classpath ./:"); + File jarsPathF = new File(jarsPath); + File[] files = jarsPathF.listFiles(); + + for (File jar : files) { + + if (jar.getName().endsWith(".jar")) { + sb.append("./" + jar.getName()); + sb.append(":"); + } + } + + sb.deleteCharAt(sb.length() - 1); + sb.append(" " + fullMainClass + " $2 " + jobOutput); + sb.append("\n"); + + AnalysisLogger.getLogger().trace("D4ScienceGenerator->Generating script in " + expectedscript.getAbsolutePath()); + FileTools.saveString(expectedscript.getAbsolutePath(), sb.toString(), true, "UTF-8"); + } + AnalysisLogger.getLogger().trace("D4ScienceGenerator->Script " + expectedscript.getAbsolutePath()+" yet exists!"); + } + + public String getResources() { + Resources res = new Resources(); + try { + int activeNodes = jobManager.getActiveNodes(); + for (int i = 0; i < activeNodes; i++) { + try { + res.addResource("Worker_" + (i + 1), 100); + } catch (Exception e1) { + } + } + } catch (Exception e) { + AnalysisLogger.getLogger().debug("D4ScienceGenerator->active nodes not ready"); + } + if ((res != null) && (res.list != null)) + return HttpRequest.toJSon(res.list).replace("resId", "resID"); + else + return ""; + } + + public float getStatus() { + try { + if (stop) + return 100f; + else + if (jobManager!=null) + return Math.max(0.5f, jobManager.getStatus() * 100f); + else + return 0; + } catch (Exception e) { + return 0f; + } + } + + public ALG_PROPS[] getSupportedAlgorithms() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON}; + return p; + } + + public INFRASTRUCTURE getInfrastructure() { + return INFRASTRUCTURE.D4SCIENCE; + } + + public void shutdown() { + + try { + jobManager.stop(); + } catch (Exception e) { + } + stop = true; + } + + public String getLoad() { + long tk = System.currentTimeMillis(); + ResourceLoad rs = null; + if (jobManager!=null) + rs = new ResourceLoad(tk, jobManager.currentNumberOfStages*subdivisiondiv); + else + rs = new ResourceLoad(tk, 0); + return rs.toString(); + } + + private long lastTime; + private int lastProcessed; + public String getResourceLoad() { + long thisTime = System.currentTimeMillis(); + int processedRecords = 0; + if ((jobManager!=null) && (subdivisiondiv>0)) + processedRecords = jobManager.currentNumberOfStages*subdivisiondiv; + + int estimatedProcessedRecords = 0; + if (processedRecords == lastProcessed) { + estimatedProcessedRecords = Math.round(((float) thisTime * (float) lastProcessed) / (float) lastTime); + } else { + lastProcessed = processedRecords; + estimatedProcessedRecords = lastProcessed; + } + lastTime = thisTime; + ResourceLoad rs = new ResourceLoad(thisTime, estimatedProcessedRecords); + return rs.toString(); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/job/management/QueueJobManager.java b/src/main/java/org/gcube/dataanalysis/executor/job/management/QueueJobManager.java new file mode 100644 index 0000000..e488fb8 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/job/management/QueueJobManager.java @@ -0,0 +1,821 @@ +package org.gcube.dataanalysis.executor.job.management; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.UUID; + +import javax.jms.ExceptionListener; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageListener; + +import org.apache.activemq.ActiveMQConnection; +import org.apache.axis.message.addressing.EndpointReferenceType; +import org.gcube.common.core.contexts.GHNContext; +import org.gcube.common.core.informationsystem.client.AtomicCondition; +import org.gcube.common.core.informationsystem.client.ISClient; +import org.gcube.common.core.informationsystem.client.RPDocument; +import org.gcube.common.core.informationsystem.client.queries.WSResourceQuery; +import org.gcube.common.core.scope.GCUBEScope; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.resource.StorageObject; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; +import org.gcube.dataanalysis.ecoengine.utils.Operations; +import org.gcube.dataanalysis.executor.messagequeue.ATTRIBUTE; +import org.gcube.dataanalysis.executor.messagequeue.Consumer; +import org.gcube.dataanalysis.executor.messagequeue.Producer; +import org.gcube.dataanalysis.executor.messagequeue.QCONSTANTS; +import org.gcube.dataanalysis.executor.messagequeue.QueueManager; +import org.gcube.dataanalysis.executor.scripts.ScriptIOWorker; +import org.gcube.vremanagement.executor.stubs.ExecutorCall; +import org.gcube.vremanagement.executor.stubs.TaskCall; +import org.gcube.vremanagement.executor.stubs.TaskProxy; + +public class QueueJobManager { + + // broadcast message period + public static int broadcastTimePeriod = 120000; + // max silence before computation stops + public static int maxSilenceTimeBeforeComputationStop = 10800000; + // max number of retries per computation step + public static int maxNumberOfComputationRetries = 1; + // period for controlling a node activity + public static int computationWatcherTimerPeriod = 120000; + // max number of message to put in a queue +// protected static int maxNumberOfMessages = 20; + public static int maxNumberOfStages = Integer.MAX_VALUE;//10; + // timeout for resending a message + public static int queueWatcherMaxwaitingTime = QCONSTANTS.refreshStatusTime;// * 5; + + protected int maxFailureTries; + private static String pluginName = "generic-worker";//"GenericWorker"; + + protected String scope; + protected GCUBEScope gscope; + protected String session; + + protected boolean yetstopped; + protected boolean messagesresent; + protected float status; + protected boolean abort; + protected boolean shutdown; + + protected List eprs; + protected int activeNodes; + protected int computingNodes; + protected int numberOfMessages; + protected int totalNumberOfMessages; + protected int actualNumberOfNodes; + protected int totalNumberOfStages; + public int currentNumberOfStages; + + // files management + protected List filenames; + protected List fileurls; + + // queue parameters + protected String queueName; + protected String queueResponse; + protected String queueURL; + protected String queueUSER; + protected String queuePWD; + protected org.gcube.dataanalysis.executor.messagequeue.Consumer consumer; + protected Producer producer; + + Timer broadcastTimer; + Timer computationWatcherTimer; + ComputationTimerWatcher computationWatcher; + String serviceClass; + String serviceName; + String owner; + String localDir; + String remoteDir; + String outputDir; + String script; + List arguments; + String configuration; + boolean deletefiles; + StatusListener statuslistener; + + private void resetAllVars() { + scope = null; + gscope = null; + + yetstopped = false; + messagesresent = false; + status = 0; + abort = false; + shutdown = false; + + eprs = null; + activeNodes = 0; + computingNodes = 0; + numberOfMessages = 0; + + actualNumberOfNodes = 0; + filenames = null; + fileurls = null; + + queueName = null; + queueResponse = null; + queueURL = null; + queueUSER = null; + queuePWD = null; + consumer = null; + producer = null; + broadcastTimer = null; + computationWatcherTimer = null; + computationWatcher = null; + serviceClass = null; + serviceName = null; + owner = null; + localDir = null; + remoteDir = null; + outputDir = null; + script = null; + arguments = null; + configuration = null; + deletefiles = false; + statuslistener = null; + } + + public int getActiveNodes() { + return computingNodes; + } + + public float getStatus() { + float innerStatus = 0; + if (totalNumberOfMessages != 0) + innerStatus = (1f - ((float) numberOfMessages / (float) totalNumberOfMessages)); + if (totalNumberOfStages == 0) + return innerStatus; + else { + float offset = ((float) Math.max(currentNumberOfStages - 1, 0)) / (float) totalNumberOfStages; + float status = offset + (innerStatus / (float) totalNumberOfStages); + // AnalysisLogger.getLogger().info("stages: "+totalNumberOfStages+" inner status: "+innerStatus+" currentStage: "+currentNumberOfStages+" status: "+status); + return status; + } + } + + // there is only one node from the client point of view + public int getNumberOfNodes() { + if (eprs.size() > 0) + return 1; + else + return 0; + } + + public void setNumberOfNodes(int newNumberOfNodes) { + // ignore this setting in this case + } + + private void init(String scope, int numberOfNodes) throws Exception { + resetAllVars(); + // init scope variables + this.scope = scope; + gscope = GCUBEScope.getScope(scope); + // introduce a session + // initialize flags + shutdown = false; + yetstopped = false; + messagesresent = false; + abort = false; + // find all the nodes - initialize the eprs + findNodes(scope); + } + + public QueueJobManager(String scope, int numberOfNodes) throws Exception { + init(scope, numberOfNodes); + } + + public QueueJobManager(String scope, int numberOfNodes, List eprs) throws Exception { + init(scope, numberOfNodes); + this.eprs = eprs; + } + + private void setGlobalVars(String serviceClass, String serviceName, String owner, String localDir, String remoteDir, String outputDir, String script, List arguments, String configuration, boolean deletefiles) { + this.serviceClass = serviceClass; + this.serviceName = serviceName; + this.owner = owner; + this.localDir = localDir; + this.remoteDir = remoteDir; + this.outputDir = outputDir; + this.script = script; + this.arguments = arguments; + this.configuration = configuration; + this.deletefiles = deletefiles; + } + + private int totalmessages = 0; + + public boolean uploadAndExecuteChunkized(String serviceClass, String serviceName, String owner, String localDir, String remoteDir, String outputDir, String script, List arguments, String configuration, boolean deletefiles, boolean forceUpload) throws Exception { + long t0 = System.currentTimeMillis(); + + int elements = arguments.size(); + /* + * int div = elements / (maxNumberOfMessages); int rest = elements % (maxNumberOfMessages); if (rest > 0) div++; if (div == 0) { div = 1; } + */ + session = (("" + UUID.randomUUID()).replace("-", "") + Math.random()).replace(".", ""); + int[] chunkSizes = null; + //up to 1120 species we don't make stages + if (elements>maxNumberOfStages) + chunkSizes = Operations.takeChunks(elements, maxNumberOfStages); + else { + chunkSizes = new int[1]; + chunkSizes[0]=elements; + } + int allchunks = chunkSizes.length; + totalNumberOfStages = allchunks; + currentNumberOfStages = 0; + int start = 0; + totalmessages = 0; + AnalysisLogger.getLogger().info("Starting the computation in "+allchunks+" stages"); + for (int i = 0; i < allchunks; i++) { + numberOfMessages = totalNumberOfMessages = 0; + currentNumberOfStages++; + int end = Math.min(elements, start + chunkSizes[i]); + AnalysisLogger.getLogger().info("Computing the chunk number " + (i + 1) + " of " + allchunks + " between " + start + " and " + (end - 1)); + List sublist = new ArrayList(); + for (int j = start; j < end; j++) + sublist.add(arguments.get(j)); + + AnalysisLogger.getLogger().info("size sub:" + sublist.size()); + // totalmessages=totalmessages+sublist.size(); + uploadAndExecute(serviceClass, serviceName, owner, localDir, remoteDir, outputDir, script, sublist, configuration, deletefiles, forceUpload); + if (abort) + break; + start = end; + AnalysisLogger.getLogger().info("Processed chunk number " + (i + 1)); + + } + + currentNumberOfStages = totalNumberOfStages; + AnalysisLogger.getLogger().info("Finished computation on all chunks and messages " + totalmessages); + AnalysisLogger.getLogger().info("Whole Procedure done in " + (System.currentTimeMillis() - t0) + " ms"); + return (!abort); + } + + private boolean uploadAndExecute(String serviceClass, String serviceName, String owner, String localDir, String remoteDir, String outputDir, String script, List arguments, String configuration, boolean deletefiles, boolean forceUpload) throws Exception { + int numberOfRetries = maxNumberOfComputationRetries; + boolean recompute = true; + + while ((numberOfRetries > 0) && (recompute)) { + long t0 = System.currentTimeMillis(); + // if (numberOfRetries inputs = generateInputMessage(filenames, fileurls, outputDir, script, arguments.get(k), k, scope, serviceClass, serviceName, owner, remoteDir, session, configuration, deletefiles); producer.sendMessage(inputs, 0); AnalysisLogger.getLogger().info("Sent Message " + k); } } waitForMessages(); if (numberOfMessages>0){ abort = true; } } + */ + + // deleteRemoteFolder(); + // summary + AnalysisLogger.getLogger().info("-SUMMARY-"); + for (int i = 0; i < totalNumberOfMessages; i++) { + if (activeMessages[i]) + AnalysisLogger.getLogger().info("Error : the Message Number " + i + " Was Never Processed!"); + if (resentMessages[i] > 0) { + messagesresent = true; + AnalysisLogger.getLogger().info("Warning : the Message Number " + i + " Was resent " + resentMessages[i] + " Times"); + } + } + AnalysisLogger.getLogger().info("-SUMMARY END-"); + + stop(); + AnalysisLogger.getLogger().info("Stopped"); + AnalysisLogger.getLogger().info("Single Step Procedure done in " + (System.currentTimeMillis() - t0) + " ms"); + activeNodes = 0; + numberOfRetries--; + if (abort) { + recompute = true; + if (numberOfRetries > 0) + Thread.sleep(10000); + } else + recompute = false; + } + + return (!abort); + } + + public boolean hasResentMessages() { + return messagesresent; + } + + public void waitForMessages() throws Exception { + AnalysisLogger.getLogger().info("Waiting..."); + while ((numberOfMessages > 0) && (!abort)) { + Thread.sleep(2000); + + // long tcurrent = System.currentTimeMillis(); + // if ((tcurrent - waitTime) > maxwaitingTime) { + // break; + // } + } + AnalysisLogger.getLogger().info("...Stop - Abort?" + abort); + } + + public boolean wasAborted() { + return abort; + } + + public void purgeQueues() throws Exception { + AnalysisLogger.getLogger().info("Purging Queue"); + List tasksProxies = new ArrayList(); + for (int j = 0; j < actualNumberOfNodes; j++) { + try { + contactNodes(tasksProxies, j, queueName, queueUSER, queuePWD, queueURL, queueResponse, session, "true"); + } catch (Exception e) { + e.printStackTrace(); + AnalysisLogger.getLogger().info("Error in purgin queue on node " + j); + } + } + AnalysisLogger.getLogger().info("Queue Purged"); + } + + public void stop() { + try { + if (!yetstopped) { + if (broadcastTimer != null) { + AnalysisLogger.getLogger().info("Stopping Broadcaster"); + broadcastTimer.cancel(); + broadcastTimer.purge(); + } + + if (computationWatcherTimer != null) { + AnalysisLogger.getLogger().info("Stopping Watcher"); + computationWatcherTimer.cancel(); + computationWatcherTimer.purge(); + } + + AnalysisLogger.getLogger().info("Purging Status Listener"); + + if (statuslistener != null) + statuslistener.destroyAllWatchers(); + + AnalysisLogger.getLogger().info("Stopping Producer and Consumer"); + + try{ + producer.stop(); + producer.closeSession(); + }catch(Exception e1){} + try{ + consumer.stop(); + consumer.closeSession(); + }catch(Exception e2){} + + AnalysisLogger.getLogger().info("Purging Remote Queues"); + purgeQueues(); + + yetstopped = true; + } + } catch (Exception e) { + e.printStackTrace(); + AnalysisLogger.getLogger().info("Not completely stopped"); + } + } + + private void contactNodes(List tasksProxies, int order, String queueName, String queueUSER, String queuePWD, String queueURL, String queueResponse, String session, String purgeQueue) throws Exception { + // generate the input map according to the arguments + Map inputs = generateWorkerInput(queueName, queueUSER, queuePWD, queueURL, queueResponse, session, purgeQueue); + AnalysisLogger.getLogger().info("Inputs " + inputs); + // take the i-th endpoint of the executor + EndpointReferenceType selectedEPR = eprs.get(order); + AnalysisLogger.getLogger().info("Broadcasting to node " + (order + 1) + " on " + selectedEPR.getAddress()); + // run the executor script + ExecutorCall call = new ExecutorCall(pluginName, gscope); + call.setEndpointReference(selectedEPR); + TaskCall task = null; + AnalysisLogger.getLogger().info("EPR:" + selectedEPR); + task = call.launch(inputs); +// AnalysisLogger.getLogger().info("Task EPR:" + task.getEndpointReference()); + TaskProxy proxy = task.getProxy(); + tasksProxies.add(new WorkerWatcher(proxy, AnalysisLogger.getLogger())); + // AnalysisLogger.getLogger().info("Contacting node " + (order + 1) + " OK on " + selectedEPR); + + } + + private int findNodes(String scopeString) throws Exception { + AnalysisLogger.getLogger().debug("SCOPE:"+scopeString); + GCUBEScope scope = GCUBEScope.getScope(scopeString); + ISClient client = GHNContext.getImplementation(ISClient.class); + WSResourceQuery wsquery = client.getQuery(WSResourceQuery.class); + wsquery.addAtomicConditions(new AtomicCondition("//gc:ServiceName", "Executor")); + wsquery.addAtomicConditions(new AtomicCondition("/child::*[local-name()='Task']/name[text()='" + pluginName + "']", pluginName)); + List listdoc = client.execute(wsquery, scope); + EndpointReferenceType epr = null; + eprs = new ArrayList(); + int numberOfEP = 0; + for (RPDocument resource : listdoc) { + epr = resource.getEndpoint(); + numberOfEP++; + eprs.add(epr); + } + AnalysisLogger.getLogger().info("Found " + numberOfEP + " endpoints"); + // get current number of available nodes + actualNumberOfNodes = eprs.size(); + return numberOfEP; + } + + private void setQueueVariables() throws Exception { + queueName = "D4ScienceJob"; // + session; + queueResponse = queueName + "Response"+session; + //general scope + queueURL = gscope.getServiceMap().getEndpoints(GHNContext.MSGBROKER).iterator().next().getAddress().toString(); + //tests on ecosystem + //TODO: delete this! +// queueURL = "tcp://ui.grid.research-infrastructures.eu:6166"; +// queueURL = "tcp://message-broker.d4science.research-infrastructures.eu:6166"; + AnalysisLogger.getLogger().info("Queue for the scope: " + queueURL); + if (queueURL==null){ + if (scope.startsWith("/gcube")) + queueURL = "tcp://ui.grid.research-infrastructures.eu:6166"; + else + queueURL = "tcp://message-broker.d4science.research-infrastructures.eu:6166"; + } + queueUSER = ActiveMQConnection.DEFAULT_USER; + queuePWD = ActiveMQConnection.DEFAULT_PASSWORD; + } + + public void deleteRemoteFolder() throws Exception { + ScopeProvider.instance.set(scope); + IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED,MemoryType.VOLATILE).getClient(); +// IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, gscope).getClient(); + AnalysisLogger.getLogger().info("Removing Remote Dir " + remoteDir); + client.removeDir().RDir(remoteDir); + AnalysisLogger.getLogger().info("Removed"); + } + + private void uploadFilesOnStorage(boolean forceupload) throws Exception { + ScopeProvider.instance.set(scope); + IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, MemoryType.VOLATILE).getClient(); +// IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, gscope).getClient(); + File dir = new File(localDir); + File[] files = dir.listFiles(); + AnalysisLogger.getLogger().info("Start uploading"); + filenames = new ArrayList(); + fileurls = new ArrayList(); + boolean uploadFiles = forceupload; + // if we do not force upload then check if the folder is yet there + if (!uploadFiles) { + List remoteObjects = client.showDir().RDir(remoteDir); + // only upload files if they are not yet uploaded + if (remoteObjects.size() == 0) + uploadFiles = true; + } + if (!uploadFiles) + AnalysisLogger.getLogger().info("Unnecessary to Uploading Files"); + + AnalysisLogger.getLogger().info("Loading files"); + for (File sfile : files) { + if (sfile.getName().startsWith(".")) + continue; + + String localf = sfile.getAbsolutePath(); + String filename = sfile.getName(); + String remotef = remoteDir + sfile.getName(); + if (uploadFiles) { + client.put(true).LFile(localf).RFile(remotef); + AnalysisLogger.getLogger().info("Uploading File "+localf+" as remote file "+remotef); + } + String url = client.getUrl().RFile(remotef); +// AnalysisLogger.getLogger().info("URL obtained: " + url); + filenames.add(filename); + fileurls.add(url); + } + AnalysisLogger.getLogger().info("Loading finished"); + + } + + private void broadcastListenCommandToExecutorNodes() throws Exception { + AnalysisLogger.getLogger().info("Submitting script to Remote Queue " + queueName); + List tasksProxies = new ArrayList(); + try{ + findNodes(scope); + }catch(Exception e){ + AnalysisLogger.getLogger().info("Error in Finding nodes - using previous value"); + } + activeNodes = actualNumberOfNodes; + // launch the tasks + for (int i = 0; i < actualNumberOfNodes; i++) { + try { + contactNodes(tasksProxies, i, queueName, queueUSER, queuePWD, queueURL, queueResponse, session, "false"); + } catch (Exception e) { + e.printStackTrace(); + AnalysisLogger.getLogger().info("Error in Contacting nodes"); + } + } + } + + private void createClientProducer() throws Exception { + AnalysisLogger.getLogger().info("Creating Message Queue and Producer"); + // create the Producer + QueueManager qm = new QueueManager(); + qm.createAndConnect(queueUSER, queuePWD, queueURL, queueName); + producer = new Producer(qm, queueName); + AnalysisLogger.getLogger().info("Producer OK"); + } + + private void createClientConsumer() throws Exception { + AnalysisLogger.getLogger().info("Creating Response Message Queue and Consumer"); + // create the listener + statuslistener = new StatusListener(); + QueueManager qm1 = new QueueManager(); + qm1.createAndConnect(queueUSER, queuePWD, queueURL, queueResponse); + consumer = new Consumer(qm1, statuslistener, statuslistener, queueResponse); + AnalysisLogger.getLogger().info("Consumers OK"); + } + + boolean activeMessages[]; + public int resentMessages[]; + + private void sendMessages() throws Exception { + int i = 0; + numberOfMessages = arguments.size(); + totalNumberOfMessages = numberOfMessages; + AnalysisLogger.getLogger().info("Messages To Send " + numberOfMessages); + activeMessages = new boolean[numberOfMessages]; + resentMessages = new int[numberOfMessages]; + for (String argum : arguments) { + Map inputs = generateInputMessage(filenames, fileurls, outputDir, script, argum, i, scope, serviceClass, serviceName, owner, remoteDir, session, configuration, deletefiles, false); + producer.sendMessage(inputs, 0); + AnalysisLogger.getLogger().info("Send " + i); + activeMessages[i] = true; + i++; + } + AnalysisLogger.getLogger().info("Messages Sent " + numberOfMessages); + } + + private Map generateInputMessage(Object filenames, Object fileurls, String outputDir, String script, String argum, int i, String scope, String serviceClass, String serviceName, String owner, String remoteDir, String session, String configuration, boolean deletefiles, boolean duplicateMessage) { + Map inputs = new HashMap(); + + inputs.put(ATTRIBUTE.FILE_NAMES.name(), filenames); + inputs.put(ATTRIBUTE.FILE_URLS.name(), fileurls); + inputs.put(ATTRIBUTE.OUTPUTDIR.name(), outputDir); + inputs.put(ATTRIBUTE.SCRIPT.name(), script); + inputs.put(ATTRIBUTE.ARGUMENTS.name(), argum + " " + duplicateMessage); + inputs.put(ATTRIBUTE.ORDER.name(), "" + i); + inputs.put(ATTRIBUTE.SCOPE.name(), scope); + inputs.put(ATTRIBUTE.SERVICE_CLASS.name(), serviceClass); + inputs.put(ATTRIBUTE.SERVICE_NAME.name(), serviceName); + inputs.put(ATTRIBUTE.OWNER.name(), owner); + inputs.put(ATTRIBUTE.REMOTEDIR.name(), remoteDir); + inputs.put(ATTRIBUTE.CLEAN_CACHE.name(), "" + deletefiles); + inputs.put(ATTRIBUTE.QSESSION.name(), session); + inputs.put(ATTRIBUTE.CONFIGURATION.name(), configuration); + inputs.put(ATTRIBUTE.TOPIC_RESPONSE_NAME.name(), queueResponse); + inputs.put(ATTRIBUTE.QUEUE_USER.name(), queueUSER); + inputs.put(ATTRIBUTE.QUEUE_PASSWORD.name(), queuePWD); + inputs.put(ATTRIBUTE.QUEUE_URL.name(), queueURL); + return inputs; + } + + private Map generateWorkerInput(String queueName, String queueUser, String queuePassword, String queueURL, String queueResponse, String session, String purge) { + + Map inputs = new HashMap(); + + inputs.put(ATTRIBUTE.TOPIC_NAME.name(), ScriptIOWorker.toInputString(queueName)); + inputs.put(ATTRIBUTE.QUEUE_USER.name(), ScriptIOWorker.toInputString(queueUser)); + inputs.put(ATTRIBUTE.QUEUE_PASSWORD.name(), ScriptIOWorker.toInputString(queuePassword)); + inputs.put(ATTRIBUTE.QUEUE_URL.name(), ScriptIOWorker.toInputString(queueURL)); + inputs.put(ATTRIBUTE.TOPIC_RESPONSE_NAME.name(), ScriptIOWorker.toInputString(queueResponse)); + inputs.put(ATTRIBUTE.QSESSION.name(), session); + inputs.put(ATTRIBUTE.ERASE.name(), purge); + return inputs; + } + + public class Broadcaster extends TimerTask { + + @Override + public void run() { + try { + AnalysisLogger.getLogger().info("(((((((((((((((((((((((((((------Broadcasting Information To Watchers------)))))))))))))))))))))))))))"); + broadcastListenCommandToExecutorNodes(); + AnalysisLogger.getLogger().info("(((((((((((((((((((((((((((------END Broadcasting Information To Watchers------)))))))))))))))))))))))))))"); + } catch (Exception e) { + e.printStackTrace(); + AnalysisLogger.getLogger().info("--------------------------------Broadcaster: Error Sending Listen Message to Executors------)))))))))))))))))))))))))))"); + } + } + + } + + public class ComputationTimerWatcher extends TimerTask { + + long maxTime; + long lastTimeClock; + + public ComputationTimerWatcher(long maxtime) { + this.maxTime = maxtime; + this.lastTimeClock = System.currentTimeMillis(); + } + + public void reset() { + lastTimeClock = System.currentTimeMillis(); + } + + public void setmaxTime(long maxTime) { + this.maxTime = maxTime; + } + + @Override + public void run() { + try { + long t0 = System.currentTimeMillis(); + AnalysisLogger.getLogger().info("Computation Watcher Timing Is " + (t0 - lastTimeClock)+" max computation time is "+maxTime); + if ((t0 - lastTimeClock) > maxTime) { + AnalysisLogger.getLogger().info("Computation Watcher - Computation Timeout: Closing Queue Job Manager!!!"); + abort(); + } + } catch (Exception e) { + e.printStackTrace(); + AnalysisLogger.getLogger().info("Error Taking clock"); + } + } + + } + + public synchronized void abort() { + AnalysisLogger.getLogger().info("Computation Aborted"); + this.abort = true; + } + + public class StatusListener implements MessageListener, ExceptionListener { + + private QueueWorkerWatcher[] watchers; + + synchronized public void onException(JMSException ex) { + abort(); + AnalysisLogger.getLogger().info("JMS Exception occured. Shutting down client."); + } + + private synchronized void addWatcher(int order) { + if (watchers == null) + watchers = new QueueWorkerWatcher[totalNumberOfMessages]; + + QueueWorkerWatcher watcher = watchers[order]; + if (watcher != null) { + destroyWatcher(order); + } + + Map message = generateInputMessage(filenames, fileurls, outputDir, script, arguments.get(order), order, scope, serviceClass, serviceName, owner, remoteDir, session, configuration, deletefiles, true); + watchers[order] = new QueueWorkerWatcher(producer, message, order); + } + + private synchronized void resetWatcher(int order) { + if (watchers == null) + watchers = new QueueWorkerWatcher[totalNumberOfMessages]; + else if (watchers[order] != null) + watchers[order].resetTime(); + } + + private synchronized void destroyWatcher(int order) { + if (watchers != null && watchers[order] != null) { + if (watchers[order].hasResent()) + resentMessages[order] = resentMessages[order] + 1; + + watchers[order].destroy(); + watchers[order] = null; + AnalysisLogger.getLogger().info("Destroyed Watcher number " + order); + } + } + + public synchronized void destroyAllWatchers() { + if (watchers != null) { + for (int i = 0; i < watchers.length; i++) { + destroyWatcher(i); + } + } + } + + public void onMessage(Message message) { + + // get message + try { + + HashMap details = (HashMap) (HashMap) message.getObjectProperty(ATTRIBUTE.CONTENT.name()); + String status = (String) details.get(ATTRIBUTE.STATUS.name()); + String order = "" + details.get(ATTRIBUTE.ORDER.name()); + String nodeaddress = (String) details.get(ATTRIBUTE.NODE.name()); + String msession = (String) details.get(ATTRIBUTE.QSESSION.name()); + Object error = details.get(ATTRIBUTE.ERROR.name()); + + AnalysisLogger.getLogger().info("Current session " + session); + if ((msession != null) && (msession.equals(session))) { + AnalysisLogger.getLogger().info("Session " + session + " is right - acknowledge"); + message.acknowledge(); + AnalysisLogger.getLogger().info("Session " + session + " acknowledged"); + int orderInt = -1; + try { + orderInt = Integer.parseInt(order); + } catch (Exception e3) { + e3.printStackTrace(); + } + if (orderInt > -1) { + + // reset the watcher + if (computationWatcher!=null) + computationWatcher.reset(); + + AnalysisLogger.getLogger().info("Task number " + order + " is " + status + " on node " + nodeaddress + " and session " + session); + + if (status.equals(ATTRIBUTE.STARTED.name())) { + computingNodes++; + addWatcher(orderInt); + } + if (status.equals(ATTRIBUTE.PROCESSING.name())) { + + resetWatcher(orderInt); + } else if (status.equals(ATTRIBUTE.FINISHED.name())) { + + totalmessages++; + computingNodes--; + destroyWatcher(orderInt); + if (numberOfMessages > 0) + numberOfMessages--; + + AnalysisLogger.getLogger().info("Remaining " + numberOfMessages + " messages to manage"); + activeMessages[orderInt] = false; + + } else if (status.equals(ATTRIBUTE.FATAL_ERROR.name())) { + if (error!=null) + AnalysisLogger.getLogger().info("REPORTED FATAL_ERROR on " +nodeaddress+" : "); + AnalysisLogger.getLogger().info(error); + + computingNodes--; + if (maxFailureTries <= 0) { + AnalysisLogger.getLogger().info("Too much Failures - Aborting"); + destroyAllWatchers(); + abort(); + } else { + AnalysisLogger.getLogger().info("Failure Occurred - Now Resending Message " + orderInt); + resentMessages[orderInt] = resentMessages[orderInt] + 1; + maxFailureTries--; + // resend message + Map retrymessage = generateInputMessage(filenames, fileurls, outputDir, script, arguments.get(orderInt), orderInt, scope, serviceClass, serviceName, owner, remoteDir, session, configuration, deletefiles, true); + producer.sendMessage(retrymessage, QCONSTANTS.timeToLive); + AnalysisLogger.getLogger().info("Failure Occurred - Resent Message " + orderInt); + } + + } + + } else + AnalysisLogger.getLogger().info("Ignoring message " + order + " with status " + status); + } else { + AnalysisLogger.getLogger().info("wrong session " + msession + " ignoring message"); +// consumer.manager.session.recover(); + } + } catch (Exception e) { + + AnalysisLogger.getLogger().info("Error reading details ", e); + AnalysisLogger.getLogger().info("...Aborting Job..."); + abort(); + + } + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/job/management/QueueWorkerWatcher.java b/src/main/java/org/gcube/dataanalysis/executor/job/management/QueueWorkerWatcher.java new file mode 100644 index 0000000..e7b0dc0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/job/management/QueueWorkerWatcher.java @@ -0,0 +1,76 @@ +package org.gcube.dataanalysis.executor.job.management; + +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; + +import javax.jms.Message; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.executor.messagequeue.ATTRIBUTE; +import org.gcube.dataanalysis.executor.messagequeue.Producer; +import org.gcube.dataanalysis.executor.messagequeue.QCONSTANTS; + +public class QueueWorkerWatcher { + + protected int maxwaitingTime = 2*QueueJobManager.queueWatcherMaxwaitingTime; + private long lastTimeClock; + Timer watcher; + Producer producer; + Map message; + public boolean resent=false; + int order; + + public QueueWorkerWatcher(Producer producer, Map message, int order) { + this.producer = producer; + this.message = message; + resent=false; + this.order = order; + + watcher = new Timer(); + watcher.schedule(new Controller(), 0, QCONSTANTS.refreshStatusTime); + resetTime(); + } + + public synchronized void resetTime() { + lastTimeClock = System.currentTimeMillis(); + } + + public synchronized void destroy() { + if (watcher != null) { + watcher.cancel(); + watcher.purge(); + watcher = null; + } + } + + public boolean hasResent(){ + return resent; + } + + private class Controller extends TimerTask { + + @Override + public void run() { + try { + long t0 = System.currentTimeMillis(); + AnalysisLogger.getLogger().debug("Watcher "+order+" Timing Is "+(t0 - lastTimeClock)+ " max waiting time: "+maxwaitingTime); + if ((t0 - lastTimeClock) > maxwaitingTime) { + + AnalysisLogger.getLogger().info("Watcher "+order+" Time Is Over "+(t0 - lastTimeClock)); + + AnalysisLogger.getLogger().info("Watcher "+order+" Re-Sending Message "+message); + producer.sendMessage(message, QCONSTANTS.timeToLive); +// QueueJobManager.resentMessages[Integer.parseInt(""+message.get(ATTRIBUTE.ORDER.name()))]=QueueJobManager.resentMessages[Integer.parseInt(""+message.get(ATTRIBUTE.ORDER.name()))]+1; + resent = true; + AnalysisLogger.getLogger().info("Watcher "+order+" Destroying watcher"); + destroy(); + + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/job/management/RemoteJobManager.java b/src/main/java/org/gcube/dataanalysis/executor/job/management/RemoteJobManager.java new file mode 100644 index 0000000..ed01332 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/job/management/RemoteJobManager.java @@ -0,0 +1,248 @@ +package org.gcube.dataanalysis.executor.job.management; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.axis.message.addressing.EndpointReferenceType; +import org.gcube.common.core.contexts.GHNContext; +import org.gcube.common.core.informationsystem.client.AtomicCondition; +import org.gcube.common.core.informationsystem.client.ISClient; +import org.gcube.common.core.informationsystem.client.RPDocument; +import org.gcube.common.core.informationsystem.client.queries.WSResourceQuery; +import org.gcube.common.core.scope.GCUBEScope; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; +import org.gcube.dataanalysis.executor.scripts.ScriptIOWorker; +import org.gcube.vremanagement.executor.stubs.ExecutorCall; +import org.gcube.vremanagement.executor.stubs.TaskCall; +import org.gcube.vremanagement.executor.stubs.TaskProxy; + +public class RemoteJobManager { + + private static String pluginName = "ExecutorScript"; + private int actualNumberOfNodes; + private GCUBEScope gscope; + private List eprs; + float status; + boolean abort; + boolean shutdown; + protected int activeNodes; + String scope; + + public int getActiveNodes() { + return activeNodes; + } + + public float getStatus() { + return status; + } + + public int getNumberOfNodes() { + return actualNumberOfNodes; + } + + public void setNumberOfNodes(int newNumberOfNodes) { + actualNumberOfNodes = newNumberOfNodes; + } + + public void init(String scope, int numberOfNodes) throws Exception { + this.scope = scope; + gscope = GCUBEScope.getScope(scope); + AnalysisLogger.getLogger().debug("Using the following scope for this computation:"+gscope); + shutdown = false; + yetuploaded = false; + if (eprs == null) + actualNumberOfNodes = findNodes(scope); + else + actualNumberOfNodes = eprs.size(); + + if (numberOfNodes < actualNumberOfNodes) + actualNumberOfNodes = numberOfNodes; + + } + + public RemoteJobManager(String scope, int numberOfNodes) throws Exception { + init(scope, numberOfNodes); + } + + public RemoteJobManager(String scope, int numberOfNodes, List eprs) throws Exception { + this.eprs = eprs; + init(scope, numberOfNodes); + } + + List filenames; + List fileurls; + boolean yetuploaded; + String session; + + public boolean uploadAndExecute(String serviceClass, String serviceName, String owner, String localDir, String remoteDir, String outputDir, String script, List arguments, boolean deletefiles) throws Exception { + boolean executeAll = false; + long t0 = System.currentTimeMillis(); + //if not yet uploaded , upload required files + if (!yetuploaded) { + ScopeProvider.instance.set(scope); + IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED).getClient(); +// IClient client = new StorageClient(serviceClass, serviceName, owner, AccessType.SHARED, gscope).getClient(); + File dir = new File(localDir); + File[] files = dir.listFiles(); + AnalysisLogger.getLogger().debug("Start uploading"); + filenames = new ArrayList(); + fileurls = new ArrayList(); + for (File sfile : files) { + String localf = sfile.getAbsolutePath(); + String filename = sfile.getName(); + String remotef = remoteDir + sfile.getName(); + client.put(true).LFile(localf).RFile(remotef); + String url = client.getUrl().RFile(remotef); + AnalysisLogger.getLogger().debug("URL created: " + url); + filenames.add(filename); + fileurls.add(url); + } + AnalysisLogger.getLogger().debug("Upload end"); + yetuploaded = true; + session = (""+UUID.randomUUID()).replace("-", ""); + } + + //if the number of available nodes is higher than zero launch the tasks + if (actualNumberOfNodes > 0) { + + AnalysisLogger.getLogger().debug("Executing script on " + actualNumberOfNodes + " nodes"); + int len = arguments.size(); + List tasksProxies = new ArrayList(); + activeNodes = 0; + //launch the tasks + for (int i = 0; i < actualNumberOfNodes; i++) { + String argum = ""; + //supply the arguments if they are available + if (i < len) + argum = arguments.get(i); + //generate the input map according to the arguments + Map inputs = generateInput(filenames, fileurls, outputDir, script, argum, i, scope, serviceClass, serviceName, owner, remoteDir,session,deletefiles); + AnalysisLogger.getLogger().debug("-> Owner: " + owner + " ServiceClass: " + serviceClass + " ServiceName:" + serviceName + " remoteDir:" + remoteDir); + //take the i-th endpoint of the executor + EndpointReferenceType selectedEPR = eprs.get(i); + AnalysisLogger.getLogger().debug("Launching node " + (i + 1) + " on " + selectedEPR); + //run the executor script + ExecutorCall call = new ExecutorCall(pluginName, gscope); + call.setEndpointReference(selectedEPR); + TaskCall task = null; + task = call.launch(inputs); + TaskProxy proxy = task.getProxy(); + tasksProxies.add(new WorkerWatcher(proxy, AnalysisLogger.getLogger())); + + AnalysisLogger.getLogger().debug("Launching node " + (i + 1) + " OK on " + selectedEPR); + //add the task to the list in order to reuse it + } + + activeNodes = actualNumberOfNodes; + AnalysisLogger.getLogger().debug("Launch Finished - Controlling Status"); + int allstatus = 0; + abort = false; + //control the execution: go until there are active nodes or the process must stop + while ((activeNodes != 0) && (!abort) && (!shutdown)) { + //for each node get the task state + int nworkers = tasksProxies.size(); + int i=0; + while (i < nworkers) { + WorkerWatcher proxy = tasksProxies.get(i); + String state = proxy.getState(); + AnalysisLogger.getLogger().debug("REMOTE JOB MANAGER-> STATE " + state ); + //control for aborted computation + abort = ((state == null) || state.equals("FAILED") || (!state.equals("DONE") && !state.equals("RUNNING"))); + //control for finished computation + boolean finished = false; + if (state != null) + finished = state.equals("DONE"); + //if finished update the active nodes + if (finished) { + tasksProxies.remove(i); + allstatus++; + activeNodes--; + nworkers--; + if (activeNodes == 0) + break; + } + else + i++; + + status = Math.min(((float) allstatus / (float) actualNumberOfNodes) * 100f, 95f); + if (abort) + break; + if (shutdown) + break; + // AnalysisLogger.getLogger().debug(String.format("Task " + i + "executed started at %Tc with %s state ", proxy.getStartTime(), state)); + //sleep before polling again + Thread.sleep(2000); + } + } + + activeNodes = 0; + + AnalysisLogger.getLogger().debug("All Tasks have Finished"); + if (!abort) { + AnalysisLogger.getLogger().debug("All Task were successful"); + /* + * List listElements = client.showDir().RDir(remoteDir); for (StorageObject obj : listElements) { AnalysisLogger.getLogger().debug("obj stored in directory " + remoteDir + ": " + obj.getName()); } + */ + } else + AnalysisLogger.getLogger().debug("Tasks were NOT successful"); + } else + AnalysisLogger.getLogger().debug("Warning: could not execute tasks: No Nodes Available!"); + AnalysisLogger.getLogger().debug("Whole procedure done in " + (System.currentTimeMillis() - t0) + " ms"); + status = 100f; + return executeAll; + } + + public boolean wasAborted() { + return abort; + } + + public void stop() { + shutdown = true; + } + + private int findNodes(String scopeString) throws Exception { + GCUBEScope scope = GCUBEScope.getScope(scopeString); + ISClient client = GHNContext.getImplementation(ISClient.class); + WSResourceQuery wsquery = client.getQuery(WSResourceQuery.class); + wsquery.addAtomicConditions(new AtomicCondition("//gc:ServiceName", "Executor")); + wsquery.addAtomicConditions(new AtomicCondition("/child::*[local-name()='Task']/name[text()='"+pluginName+"']", pluginName)); + List listdoc = client.execute(wsquery, scope); + EndpointReferenceType epr = null; + eprs = new ArrayList(); + int numberOfEP = 0; + for (RPDocument resource : listdoc) { + epr = resource.getEndpoint(); + numberOfEP++; + eprs.add(epr); + } + AnalysisLogger.getLogger().debug("Found " + numberOfEP + " endpoints"); + + return numberOfEP; + } + + private Map generateInput(Object filenames, Object fileurls, String outputDir, String script, String argum, int i, String scope, String serviceClass, String serviceName, String owner, String remoteDir,String session,boolean deletefiles) { + Map inputs = new HashMap(); + inputs.put("FILE_NAMES", filenames); + inputs.put("FILE_URLS", fileurls); + inputs.put("OUTPUTDIR", ScriptIOWorker.toInputString(outputDir)); + inputs.put("SCRIPT", ScriptIOWorker.toInputString(script)); + inputs.put("ARGUMENTS", ScriptIOWorker.toInputString(argum)); + inputs.put("NODE_IDENTIFIER", "" + i); + inputs.put("SCOPE", ScriptIOWorker.toInputString(scope)); + inputs.put("SERVICE_CLASS", ScriptIOWorker.toInputString(serviceClass)); + inputs.put("SERVICE_NAME", ScriptIOWorker.toInputString(serviceName)); + inputs.put("OWNER", ScriptIOWorker.toInputString(owner)); + inputs.put("REMOTEDIR", ScriptIOWorker.toInputString(remoteDir)); + inputs.put("CLEAN_CACHE",""+deletefiles); +// inputs.put("SESSION", ScriptIO.toInputString(session)); + return inputs; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/job/management/WorkerWatcher.java b/src/main/java/org/gcube/dataanalysis/executor/job/management/WorkerWatcher.java new file mode 100644 index 0000000..3c6f69c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/job/management/WorkerWatcher.java @@ -0,0 +1,37 @@ +package org.gcube.dataanalysis.executor.job.management; + +import org.apache.log4j.Logger; +import org.gcube.vremanagement.executor.stubs.TaskProxy; + +public class WorkerWatcher { + private static int maxTries = 15; + private int currentTries; + private static String runningState = "RUNNING"; + private static String failedState = "FAILED"; + Logger logger; + + TaskProxy proxy; + public WorkerWatcher(TaskProxy proxy, Logger logger){ + this.proxy = proxy; + this.logger = logger; + currentTries = 1; + } + + public String getState(){ + String state =""; + try{ + proxy.synchronize(); + state = proxy.getState(); + return state; + }catch(Exception e){ + logger.error("Error in getting state: recover try number "+currentTries,e); + currentTries++; + if (currentTries>maxTries){ + return failedState; + } + else return runningState; + } + + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/messagequeue/ATTRIBUTE.java b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/ATTRIBUTE.java new file mode 100644 index 0000000..4e18e41 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/ATTRIBUTE.java @@ -0,0 +1,34 @@ +package org.gcube.dataanalysis.executor.messagequeue; + +public enum ATTRIBUTE { + STATUS, + DONE, + STARTED, + FINISHED, + PROCESSING, + FATAL_ERROR, + TRIVIAL_ERROR, + ORDER, + NODE, + CONTENT, + QSESSION, + TOPIC_NAME, + QUEUE_USER, + QUEUE_PASSWORD, + QUEUE_URL, + TOPIC_RESPONSE_NAME, + ERASE, + FILE_NAMES, + FILE_URLS, + CONFIGURATION, + OUTPUTDIR, + OWNER, + REMOTEDIR, + SERVICE_CLASS, + SERVICE_NAME, + SCOPE, + SCRIPT, + ARGUMENTS, + CLEAN_CACHE, + ERROR +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/messagequeue/Consumer.java b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/Consumer.java new file mode 100644 index 0000000..462bf4c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/Consumer.java @@ -0,0 +1,62 @@ +package org.gcube.dataanalysis.executor.messagequeue; + +import javax.jms.ExceptionListener; +import javax.jms.JMSException; +import javax.jms.MessageConsumer; +import javax.jms.MessageListener; + +public class Consumer { + public QueueManager manager; + public MessageConsumer consumer; + private MessageListener consumerCallback; + private ExceptionListener errorCallback; + private String topic; + + + public Consumer(QueueManager manager, MessageListener consumerCallback, ExceptionListener errorCallback, String topic) throws JMSException, InterruptedException { + this.manager = manager; + this.consumerCallback = consumerCallback; + this.errorCallback = errorCallback; + this.topic = topic; + + create(); + } + + private void create() throws JMSException, InterruptedException { + +// Topic ConsumerTopic = manager.session.createTopic(topic); + + MessageConsumer consumer = manager.session.createConsumer(manager.destination); +// MessageConsumer consumer = manager.session.createDurableSubscriber(ConsumerTopic, "Consumer."+topic); +// MessageConsumer consumer = manager.session.createConsumer(ConsumerTopic); + manager.connection.setExceptionListener(errorCallback); + consumer.setMessageListener(consumerCallback); + + } + + public void standBy() throws JMSException { + if (consumer != null) + consumer.close(); + } + + public void wake() throws Exception { + this.create(); + } + + public void stop() throws JMSException { + if (consumer != null) { + consumer.close(); + + } +// closeSession(); + } + + public void closeSession() throws JMSException { + try { + manager.closeSession(); + manager.connection.close(); + } catch (Exception e) { + + } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/messagequeue/Producer.java b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/Producer.java new file mode 100644 index 0000000..c40cebf --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/Producer.java @@ -0,0 +1,70 @@ +package org.gcube.dataanalysis.executor.messagequeue; + +import java.util.UUID; + +import javax.jms.DeliveryMode; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageProducer; +import javax.jms.TextMessage; +import javax.jms.Topic; + +public class Producer { + + public MessageProducer producer; + public QueueManager manager; + public String topic; + public String identifier; + public Producer(QueueManager manager,String topic) throws JMSException { + this.manager = manager; + this.topic = topic; + this.identifier = "" + UUID.randomUUID(); + create(); + } + + private void create() throws JMSException { +// Topic ProducerTopic = manager.session.createTopic(topic); + producer = manager.session.createProducer(manager.destination); +// producer = manager.session.createProducer(ProducerTopic); + producer.setDeliveryMode(DeliveryMode.PERSISTENT); + } + + public void sendTextMessage(String text, long timeToLive) throws JMSException { + TextMessage message = manager.session.createTextMessage(text); + producer.setTimeToLive(timeToLive); + producer.send(message); + } + + public void sendMessage(Object toSend, long timeToLive) throws JMSException { + Message message = manager.session.createMessage(); + message.setObjectProperty(ATTRIBUTE.CONTENT.name(), toSend); + producer.setTimeToLive(timeToLive); + producer.send(message); + } + + public void standBy() throws JMSException { + producer.close(); + } + + public void wake() throws Exception { + this.create(); + } + + public void stop() throws JMSException { + if (producer != null){ + producer.close(); + + } +// closeSession(); + } + + + public void closeSession() throws JMSException { + try { + manager.closeSession(); + manager.connection.close(); + } catch (Exception e) { + + } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/messagequeue/QCONSTANTS.java b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/QCONSTANTS.java new file mode 100644 index 0000000..b2294b0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/QCONSTANTS.java @@ -0,0 +1,8 @@ +package org.gcube.dataanalysis.executor.messagequeue; + +public class QCONSTANTS { + + public static int refreshStatusTime = 60000; + public static int QueueLifeTime = 60000;//3600000; + public static long timeToLive = 0; +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/messagequeue/QueueManager.java b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/QueueManager.java new file mode 100644 index 0000000..36f6662 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/messagequeue/QueueManager.java @@ -0,0 +1,77 @@ +package org.gcube.dataanalysis.executor.messagequeue; + +import java.util.Hashtable; +import java.util.Properties; +import java.util.UUID; + +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.Session; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; +import javax.naming.Context; +import javax.naming.InitialContext; + +import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.activemq.broker.BrokerService; +import org.apache.activemq.broker.jmx.QueueViewMBean; + +public class QueueManager { + + public ActiveMQConnectionFactory connectionFactory; + public Connection connection; + public Session session; + public Destination destination; + boolean transacted = false; + public String mqurl; + private String identifier; + + public void createAndConnect(String user,String password, String mqurl, String queueName) throws JMSException { + this.mqurl=mqurl; + connect(user,password,mqurl); + session = connection.createSession(transacted, Session.CLIENT_ACKNOWLEDGE); + this.identifier = ""+UUID.randomUUID(); + /* + Hashtable properties = new Hashtable(); + properties.put(Context.INITIAL_CONTEXT_FACTORY, "org.apache.activemq.jndi.ActiveMQInitialContextFactory"); + properties.put(Context.PROVIDER_URL, mqurl); + InitialContext context = new InitialContext(properties); + ConnectionFactory factory = (ConnectionFactory) context.lookup("ConnectionFactory"); + + destination = (Destination) context.lookup(queueName); +*/ +// destination = session.createQueue(queueName+"?consumer.prefetchSize=3"); + destination = session.createQueue(queueName+"?wireFormat.maxInactivityDurationInitalDelay=3600000&requestTimeout=240000&wireFormat.maxInactivityDuration=3600000"); + } + + public void destroy(){ + + } + + private void connect(String user,String password, String mqurl) throws JMSException{ + connectionFactory = new ActiveMQConnectionFactory(user, password, mqurl); + connectionFactory.getPrefetchPolicy().setQueuePrefetch(1); + +// Properties p = new Properties(); +// p.put("persistent", "false"); +// p.put("consumer.prefetchSize", "3"); +// p.put("ms.prefetchPolicy.all", "3"); +// p.put("cms.PrefetchPolicy.queuePrefetch", "3"); + +// connectionFactory.setProperties(p); + connection = connectionFactory.createConnection(); + connection.setClientID(identifier); + connection.start(); + } + + public void closeSession() throws Exception{ +// session.unsubscribe(identifier); + + session.close(); + + } +} \ No newline at end of file diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsNative2050Node.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsNative2050Node.java new file mode 100644 index 0000000..94236d1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsNative2050Node.java @@ -0,0 +1,19 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + + +public class AquamapsNative2050Node extends AquamapsNativeNode{ + + public AquamapsNative2050Node(){ + super(); + type = "2050"; + } + + public String getName() { + return "AQUAMAPS_NATIVE_2050"; + } + + public String getDescription() { + return "Algorithm for Native Range in 2050 by Aquamaps on a single node"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsNativeNode.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsNativeNode.java new file mode 100644 index 0000000..724d4b4 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsNativeNode.java @@ -0,0 +1,70 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsNative; + +public class AquamapsNativeNode extends AquamapsSuitableNode{ + + public AquamapsNativeNode(){ + super(); + } + + public String getName() { + return "AQUAMAPS_NATIVE"; + } + + public String getDescription() { + return "Algorithm for Native Range by Aquamaps on a single node"; + } + + // writes the distribution model on the DB: input species vector + list of areas vectors to report + public void singleStepPostprocess(Object species) { + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Analyzing Species distribution"); + // write info on DB + Queue rows = new ConcurrentLinkedQueue(); + String speciesID = AquamapsSuitableFunctions.getMainInfoID(species); + Map csquaresMap = operations.completeDistribution.get(speciesID); + + if (csquaresMap != null) { + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Getting csquare probabilites"); + // write only processed areas + for (String singleCsquare : csquaresMap.keySet()) { + String additionalInformation = operations.getAdditionalInformation(species, operations.processedAreas.get(singleCsquare)); + if (additionalInformation == null) + additionalInformation = ""; + else if (additionalInformation.length() > 0) + additionalInformation = "," + additionalInformation.trim(); + + float prob = 0f; + try { + prob = csquaresMap.get(singleCsquare); + } catch (Exception e) { + System.out.println("Aquamaps Algorithm Single Step PostProcess ->Error in getting probability value at " + speciesID + " , " + singleCsquare); + } + if (prob > 0) + rows.offer("'" + speciesID + "','" + singleCsquare + "','" + MathFunctions.roundDecimal(prob, 3) + "'" + additionalInformation); + } + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Filtering probabilities. Size:"+rows.size()); + Queue newrows = new AquamapsNative().filterProbabilitySet(rows); + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Filtered probabilities. Size:"+newrows.size()); + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Writing rows on DB"); + List toWrite = new ArrayList(); + for (String row:newrows){ + toWrite.add(row); +// System.out.println("Added row: "+row); + } + AquamapsSuitableFunctions.writeOnDB(toWrite, currentconfig.getParam("DistributionTable"), dbHibConnection); + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Rows written on DB"); + } + else + System.out.println("Aquamaps Algorithm Single Step PostProcess-> Probability distribution is void"); + + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitable2050Node.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitable2050Node.java new file mode 100644 index 0000000..66a0269 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitable2050Node.java @@ -0,0 +1,19 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + + +public class AquamapsSuitable2050Node extends AquamapsSuitableNode{ + + public AquamapsSuitable2050Node(){ + super(); + type = "2050"; + } + + public String getName() { + return "AQUAMAPS_SUITABLE_2050"; + } + + public String getDescription() { + return "Algorithm for Suitable Range in 2050 by Aquamaps on a single node"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableFunctions.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableFunctions.java new file mode 100644 index 0000000..8625c29 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableFunctions.java @@ -0,0 +1,200 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.ObjectInputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsAlgorithmCore; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.hibernate.SessionFactory; + +import com.thoughtworks.xstream.XStream; + +public class AquamapsSuitableFunctions { + + public static String countAllSpeciesQuery = "select count(*) from %1$s;"; +// public static String countAll = "select count(*) from %1$s;"; + public static String countAll = "EXPLAIN SELECT * FROM %1$s;"; + public static String countCsquareCodeQuery = "select count (*) from %1$s d where oceanarea>0"; + public static String selectAllSpeciesQuery = "select depthmin,meandepth,depthprefmin,pelagic,depthprefmax,depthmax,tempmin,layer,tempprefmin,tempprefmax,tempmax,salinitymin,salinityprefmin,salinityprefmax,salinitymax,primprodmin,primprodprefmin,primprodprefmax,primprodmax,iceconmin,iceconprefmin,iceconprefmax,iceconmax,landdistyn,landdistmin,landdistprefmin,landdistprefmax,landdistmax,nmostlat,smostlat,wmostlong,emostlong,faoareas,speciesid from %1$s order by speciesid limit %2$s offset %3$s;"; + public static String csquareCodeQuery = "select csquarecode,depthmean,depthmax,depthmin, sstanmean,sbtanmean,salinitymean,salinitybmean, primprodmean,iceconann,landdist,oceanarea,centerlat,centerlong,faoaream,eezall,lme from %1$s d where oceanarea>0 order by csquarecode limit %2$s offset %3$s"; + public static String createTableStatement = "CREATE TABLE %1$s ( speciesid character varying, csquarecode character varying, probability real, boundboxyn smallint, faoareayn smallint, faoaream integer, eezall character varying, lme integer) WITH (OIDS=FALSE ) #TABLESPACE#; CREATE INDEX CONCURRENTLY %1$s_idx ON %1$s USING btree (speciesid, csquarecode, faoaream, eezall, lme);"; + public static String metainfo = "boundboxyn, faoareayn, faoaream, eezall, lme"; + public static String selectAllSpeciesObservationQuery = "SELECT speciesid,maxclat,minclat from %1$s;"; + public static String probabilityInsertionStatement = "insert into %1$s (speciesid,csquarecode,probability %ADDEDINFORMATION%) VALUES %2$s"; + public static String deleteDuplicates = "delete from %1$s where speciesid='%2$s'"; + + + // Default Files + private static String speciesFile = "species.dat"; + private static String csquaresFile = "csquares.dat"; + private static String maxminlatFile = "maxminlat.dat"; + private static String configFile = "config.dat"; + + // file1 + public HashMap> allSpeciesObservations; + // file2 + public List speciesVectors; + // file3 + public List environmentVectors; + public int numberOfSpecies; + public int numberOfCells; + + + //processing variables + public AlgorithmConfiguration currentconfig; + public HashMap currentSpeciesBoundingBoxInfo; + public String currentFAOAreas; + public AquamapsAlgorithmCore core; + public String type; + public HashMap processedAreas; + public ConcurrentHashMap> completeDistribution; + + public AquamapsSuitableFunctions(AquamapsAlgorithmCore core, String type, AlgorithmConfiguration config) { + this.core = core; + this.type = type; + this.currentconfig = config; + } + + //PROBABILITY CALCULATION + // calculates probability and takes into account the processes areas by this node + public float calcProb(Object species, Object area) { + float prob = (float) core.getSpeciesProb((Object[]) species, (Object[]) area); + + String speciesID = getMainInfoID(species); + String csquareCode = getGeographicalID(area); + if (completeDistribution == null) + completeDistribution = new ConcurrentHashMap>(); + + Map geoDistrib = completeDistribution.get(speciesID); + // if the map is null then generate a new map, otherwise update it + if (geoDistrib == null) { + geoDistrib = new ConcurrentHashMap(); + completeDistribution.put(speciesID, geoDistrib); + } + + if (prob > 0.1) { + // record the overall probability distribution + geoDistrib.put(csquareCode, prob); + if (processedAreas == null) + processedAreas = new HashMap(); + processedAreas.put(csquareCode, area); + } + + return prob; + } + + + //BOUNDING BOX CALCULATION + // calculates the bounding box information + public HashMap calculateBoundingBox(Object[] csquarecode) { + HashMap boundingInfo = core.calculateBoundingBox("" + csquarecode[0], currentSpeciesBoundingBoxInfo.get("$pass_NS"), currentSpeciesBoundingBoxInfo.get("$pass_N"), currentSpeciesBoundingBoxInfo.get("$pass_S"), AquamapsAlgorithmCore.getElement(csquarecode, 12),// centerlat + AquamapsAlgorithmCore.getElement(csquarecode, 13),// centerlong + AquamapsAlgorithmCore.getElement(csquarecode, 14),// faoaream + currentSpeciesBoundingBoxInfo.get("$paramData_NMostLat"), currentSpeciesBoundingBoxInfo.get("$paramData_SMostLat"), currentSpeciesBoundingBoxInfo.get("$paramData_WMostLong"), currentSpeciesBoundingBoxInfo.get("$paramData_EMostLong"), currentFAOAreas, currentSpeciesBoundingBoxInfo.get("$northern_hemisphere_adjusted"), currentSpeciesBoundingBoxInfo.get("$southern_hemisphere_adjusted")); + + return boundingInfo; + } + + // initializes currentFAOAreas and currentSpeciesBoundingBoxInfo + public void getBoundingBoxInformation(Object[] speciesInfoRow, Object[] speciesObservations) { + Object[] row = speciesInfoRow; + String $paramData_NMostLat = AquamapsAlgorithmCore.getElement(row, 28); + String $paramData_SMostLat = AquamapsAlgorithmCore.getElement(row, 29); + String $paramData_WMostLong = AquamapsAlgorithmCore.getElement(row, 30); + String $paramData_EMostLong = AquamapsAlgorithmCore.getElement(row, 31); + currentFAOAreas = AquamapsAlgorithmCore.getElement(row, 32); + // adjust FAO areas + currentFAOAreas = core.procFAO_2050(currentFAOAreas); + // get Bounding Box Information + // System.out.println("TYPE:"+type); + currentSpeciesBoundingBoxInfo = core.getBoundingBoxInfo($paramData_NMostLat, $paramData_SMostLat, $paramData_WMostLong, $paramData_EMostLong, speciesObservations, type); + // end of get BoundingBoxInformation + } + + + // DATABASE INTERACTION + public static void writeOnDB(List buffer, String destinationTable, SessionFactory dbHibConnection) { + + int endIndex = buffer.size(); + if (endIndex > 0) { + System.out.println("\tWriting Buffer is not empty: "+endIndex); + String $probabilityInsertionStatement = AquamapsSuitableFunctions.probabilityInsertionStatement.replace("%ADDEDINFORMATION%", ","+metainfo); + + StringBuffer sb = new StringBuffer(); + // System.out.println("writeOnDB()->PROBABILITIES BUFFER SIZE DELETION"); + for (int i = 0; i < endIndex; i++) { + sb.append("(" + buffer.get(i) + ")"); + if (i < endIndex - 1) { + sb.append(","); + } + } + + String insertionString = String.format($probabilityInsertionStatement, destinationTable, sb.toString()); + + try { +// System.out.println(insertionString); + DatabaseFactory.executeSQLUpdate(insertionString, dbHibConnection); + } catch (Exception e) { + e.printStackTrace(); + } + + } + else + System.out.println("\tWarning : writing buffer is empty!"); + System.out.println("\tWriting on DB FINISHED"); + } + + + //FILES MANAGEMENT + public void dumpAll(String path) throws Exception { + Transformations.dumpObjectToFile(path + configFile, currentconfig); +// Transformations.dumpObjectToFile(path + csquaresFile, environmentVectors); + } + + public void rebuildConfig(String configFile) throws Exception{ + FileInputStream fis = new FileInputStream(new File(configFile)); + currentconfig = (AlgorithmConfiguration) new XStream().fromXML(fis); + fis.close(); + } + // when uploaded the files will be local + public void rebuildAll(int cellOrdinal, int chunksize, int speciesOrdinal, int speciesChunkSize, String pathToFiles) throws Exception { +// currentconfig = (AlgorithmConfiguration) Transformations.getObjectFromFile(pathToFiles+configFile); + + /* + try{ + environmentVectors = (List) Transformations.getObjectFromFile(pathToFiles+csquaresFile); + }catch(Exception e){ + System.out.println("\tError in retrieving environmental vectors"); + } + */ + } + + public String getAdditionalInformation(Object species, Object area) { + Object[] arearray = (Object[]) area; + HashMap boundingInfo = calculateBoundingBox(arearray); + String addedInformation = "'" + boundingInfo.get("$InBox") + "','" + boundingInfo.get("$InFAO") + "','" + arearray[14] + "','" + arearray[15] + "','" + arearray[16] + "'"; + return addedInformation; + } + + + //AUXILIARY FUNCTIONS + public static String getMainInfoID(Object speciesInfo) { + String s = "" + ((Object[]) speciesInfo)[33]; + return s; + } + + public static String getGeographicalID(Object geoInfo) { + String s = "" + ((Object[]) geoInfo)[0]; + return s; + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableNode.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableNode.java new file mode 100644 index 0000000..3aafdc1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableNode.java @@ -0,0 +1,387 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsAlgorithmCore; +import org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsSuitable; +import org.gcube.dataanalysis.ecoengine.user.GeneratorT; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.hibernate.SessionFactory; + +public class AquamapsSuitableNode extends ActorNode { + + // variables + protected String countSpeciesQuery, countCellsQuery, createTableStatement, destinationTable, destinationTableLabel, type; + protected AquamapsAlgorithmCore core; + protected AlgorithmConfiguration currentconfig; + protected AquamapsSuitableFunctions operations; + protected SessionFactory dbHibConnection; + + public AquamapsSuitableNode() { + + } + + // Processing + // an initialization phase in which the input are initialized and then dumped to files + @Override + public void setup(AlgorithmConfiguration config) throws Exception { + AnalysisLogger.getLogger().debug("Setup of the node algorithm for AquaMaps"); + // init algorithm + long t00 = System.currentTimeMillis(); + initSingleNode(config); + long t0 = System.currentTimeMillis(); + String distributionTable = config.getParam("DistributionTable"); + // System.out.println("Dumping information .. in " + config.getPersistencePath() + config.getModel() + "/"); + // operations.dumpAll(config.getPersistencePath() + config.getModel() + "/"); + // System.out.println(".. Dump Done in " + (System.currentTimeMillis() - t0) + " ms"); + // create table + boolean createTable = config.getParam("CreateTable") != null ? Boolean.parseBoolean(config.getParam("CreateTable")) : false; + AnalysisLogger.getLogger().debug("Creating table " + distributionTable + " " + createTable); + if (config.getDatabaseDriver()==null) + config.setDatabaseDriver("org.postgresql.Driver"); + if (createTable) { + DatabaseUtils.createBigTable(createTable, distributionTable, config.getDatabaseDriver(), config.getDatabaseUserName(), config.getDatabasePassword(), config.getDatabaseURL(), createTableStatement, dbHibConnection); + } + //create minmaxlat table if necessary + AnalysisLogger.getLogger().debug("Creating min max hspen"); + createMaxMinHspen(config); + // close connection + // DatabaseUtils.closeDBConnection(dbHibConnection); + AnalysisLogger.getLogger().debug("Preparation phase finished in " + (System.currentTimeMillis() - t00) + " ms"); + + /* + * databasecheckScheduler = new Timer(); databasecheckScheduler.schedule(new DatabaseController(), 0, refreshTime); + */ + } + + + private void createMaxMinHspen(AlgorithmConfiguration config){ + + String hspenMinMaxLat=""; + if ((config.getParam("PreprocessedTable")!=null)&&(config.getParam("PreprocessedTable").length()>0)) + hspenMinMaxLat = config.getParam("PreprocessedTable"); + + //if not preprocessed then generate a preprocessed table + if ((hspenMinMaxLat==null)||(hspenMinMaxLat.length()==0)){ + //take the name of the hspen table + String hspenTable = config.getParam("EnvelopeTable"); + //check if the table exists + String supposedminmaxlattable = "maxminlat_"+hspenTable; + List select = null; + try{ + select = DatabaseFactory.executeSQLQuery("select * from "+supposedminmaxlattable+" limit 1",dbHibConnection); + }catch(Exception ee){} + //if it exists then set the table name + if (select!=null){ + hspenMinMaxLat = supposedminmaxlattable; + AnalysisLogger.getLogger().debug("Aquamaps Algorithm Node Init ->the min max latitudes table yet exists "+hspenMinMaxLat); + } + else{ + //otherwise create it by calling the creator + AnalysisLogger.getLogger().debug("Aquamaps Algorithm Node Init ->the min max latitudes table does not exist! - generating"); + hspenMinMaxLat = AquamapsSuitable.generateMaxMinHspec(supposedminmaxlattable, hspenTable,config.getParam("OccurrencePointsTable"), dbHibConnection); + AnalysisLogger.getLogger().debug("Aquamaps Algorithm Node Init ->min max latitudes table created in "+hspenMinMaxLat); + } + } + config.setParam("PreprocessedTable",hspenMinMaxLat); + } + + // initializations + public void initSingleNode(AlgorithmConfiguration config) { + initCore(config); + // take all species records + AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->Taking all species records"); + operations.numberOfSpecies = Integer.parseInt("" + (DatabaseFactory.executeSQLQuery(countSpeciesQuery, dbHibConnection)).get(0)); + // take all csquares records + AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->Count all csquares"); + operations.numberOfCells = Integer.parseInt("" + DatabaseFactory.executeSQLQuery(countCellsQuery, dbHibConnection).get(0)); + // take all csquares records + /* + * System.out.println("Aquamaps Algorithm ->Taking csquares"); String CellsQuery = String.format(AquamapsSuitableFunctions.csquareCodeQuery, currentconfig.getParam("CsquarecodesTable"), "" + operations.numberOfCells, "0"); operations.environmentVectors = DatabaseFactory.executeSQLQuery(CellsQuery, dbHibConnection); + */ + AnalysisLogger.getLogger().debug("Aquamaps Algorithm Init ->init finished"); + } + + private void initCore(AlgorithmConfiguration config) { + currentconfig = config; + AnalysisLogger.getLogger().debug("Aquamaps Algorithm-> Initializing DB connection"); + dbHibConnection = DatabaseUtils.initDBSession(config); + AnalysisLogger.getLogger().debug("Aquamaps Algorithm-> Initializing DB connection OK"); + + countSpeciesQuery = String.format(AquamapsSuitableFunctions.countAllSpeciesQuery, config.getParam("EnvelopeTable")); + countCellsQuery = String.format(AquamapsSuitableFunctions.countCsquareCodeQuery, config.getParam("CsquarecodesTable")); + + createTableStatement = String.format(AquamapsSuitableFunctions.createTableStatement, config.getParam("DistributionTable")); + if (config.getTableSpace() != null) { + createTableStatement = createTableStatement.replace("#TABLESPACE#", "TABLESPACE " + config.getTableSpace()); + } else + createTableStatement = createTableStatement.replace("#TABLESPACE#", ""); + + destinationTable = config.getParam("DistributionTable"); + destinationTableLabel = config.getParam("DistributionTableLabel"); + AnalysisLogger.getLogger().debug("Aquamaps Algorithm-> Initializing Core Algorithm"); + core = new AquamapsAlgorithmCore(); + operations = new AquamapsSuitableFunctions(core, type, config); + AnalysisLogger.getLogger().debug("Aquamaps Algorithm-> Core Initializiation OK "); + } + + // end initialization + + // the core execution of a node + @Override + public int executeNode(int cellOrdinal, int chunksize, int speciesOrdinal, int speciesChunkSize, boolean duplicate, String pathToFiles, String nodeConfigurationFileObject, String logfile) { + long t00 = System.currentTimeMillis(); + try { + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " Node Started"); + // rebuild all files + core = new AquamapsAlgorithmCore(); + operations = new AquamapsSuitableFunctions(new AquamapsAlgorithmCore(), type, null); + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " Rebuilding objects"); + long t0 = System.currentTimeMillis(); + operations.rebuildConfig(new File(pathToFiles, nodeConfigurationFileObject).getAbsolutePath()); + // operations.rebuildAll(cellOrdinal, chunksize, speciesOrdinal, speciesChunkSize, pathToFiles); + operations.currentconfig.setConfigPath(pathToFiles); + currentconfig = operations.currentconfig; + System.out.println("Aquamaps Algorithm: " + speciesOrdinal + " Objects Rebuilt in " + (System.currentTimeMillis() - t0) + " ms"); + String SpeciesQuery = String.format(AquamapsSuitableFunctions.selectAllSpeciesQuery, currentconfig.getParam("EnvelopeTable"), "" + speciesChunkSize, "" + speciesOrdinal); + String CellsQuery = String.format(AquamapsSuitableFunctions.csquareCodeQuery, currentconfig.getParam("CsquarecodesTable"), "" + chunksize, "" + cellOrdinal); + System.out.println("Aquamaps Algorithm-> Initializing DB connection with parameters: " + currentconfig.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile + " " + currentconfig.getParam("DatabaseDriver") + " " + currentconfig.getParam("DatabaseUserName") + " " + currentconfig.getParam("DatabasePassword") + " " + currentconfig.getParam("DatabaseURL")); + System.out.println("Aquamaps Algorithm-> File Exists: " + new File(currentconfig.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile).exists()); + System.out.println("Aquamaps Algorithm-> File IS READABLE: " + new File(currentconfig.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile).canRead()); + dbHibConnection = DatabaseUtils.initDBSession(currentconfig); + + System.out.println("Aquamaps Algorithm-> Building Species Observations Max-Min Latitutes"); + operations.allSpeciesObservations = new HashMap>(); + String hspenMinMaxLat = currentconfig.getParam("PreprocessedTable"); + System.out.println("Aquamaps Algorithm-> Preprocessed Table is " + hspenMinMaxLat); + System.out.println("Aquamaps Algorithm-> Query: " + String.format(AquamapsSuitableFunctions.selectAllSpeciesObservationQuery, hspenMinMaxLat)); + System.out.println("Aquamaps Algorithm-> DB CONNECTION " + dbHibConnection); + List SpeciesObservations = DatabaseFactory.executeSQLQuery(String.format(AquamapsSuitableFunctions.selectAllSpeciesObservationQuery, hspenMinMaxLat), dbHibConnection); + System.out.println("Aquamaps Algorithm-> Queried"); + if (SpeciesObservations == null) + SpeciesObservations = new ArrayList(); + System.out.println("Aquamaps Algorithm-> Obtained Observations " + SpeciesObservations.size()); + + int lenObservations = SpeciesObservations.size(); + for (int i = 0; i < lenObservations; i++) { + Object[] maxminArray = (Object[]) SpeciesObservations.get(i); + String speciesid = (String) maxminArray[0]; + List maxminInfo = new ArrayList(); + maxminInfo.add(maxminArray); + operations.allSpeciesObservations.put((String) speciesid, maxminInfo); + } + + // take all species records + System.out.println("Aquamaps Algorithm ->Taking species records"); + operations.speciesVectors = DatabaseFactory.executeSQLQuery(SpeciesQuery, dbHibConnection); + // take all csquares records + System.out.println("Aquamaps Algorithm ->Taking csquares"); + if (operations.environmentVectors == null) + operations.environmentVectors = DatabaseFactory.executeSQLQuery(CellsQuery, dbHibConnection); + + int maxSpecies = operations.speciesVectors.size(); + int maxCells = operations.environmentVectors.size(); + + System.out.println("Aquamaps Algorithm ->Processing " + maxCells + " cells VS " + maxSpecies + " species"); + + System.out.println("Aquamaps Algorithm ->Focus On Table: " + currentconfig.getParam("DistributionTable")); + for (int j = 0; j < maxSpecies; j++) { + System.out.println("Aquamaps Algorithm: calculating elements from " + cellOrdinal + " to " + maxCells + " for species " + j); + Object speciesV = operations.speciesVectors.get(j); + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " calculating probabilities"); + singleStepPreprocess(speciesV, duplicate); + for (int i = 0; i < maxCells; i++) { + Object area = operations.environmentVectors.get(i); + operations.calcProb(speciesV, area); + } + + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " probabilities calculated!"); + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " writing on DB"); + singleStepPostprocess(speciesV); + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " write on DB OK"); + } + + + + try { + FileWriter fw = new FileWriter(new File(logfile)); + fw.write("OK"); + fw.close(); + } catch (Exception e) { + + } + } catch (Exception e) { + System.err.println("ERROR " + e); + e.printStackTrace(); + } + finally{ + + DatabaseUtils.closeDBConnection(dbHibConnection); + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " Database Closed!"); + + } + System.out.println("Aquamaps Algorithm: " + cellOrdinal + " Node Ended in " + ((float) (System.currentTimeMillis() - t00) / 1000f) + " s"); + return 0; + } + + // Auxiliary Functions + // to overwrite in case of native generation in order to filer on the probabilities types + @Override + public List getInputParameters() { + List parameters = new AquamapsSuitable().getInputParameters(); + return parameters; + } + + public Queue filterProbabilitySet(Queue probabiltyRows) { + return probabiltyRows; + } + + public float getInternalStatus() { + return 100; + } + + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON}; + return p; + } + + public String getName() { + return "AQUAMAPS_SUITABLE"; + } + + public String getDescription() { + return "Algorithm by Aquamaps on a single node"; + } + + public void postProcess(boolean manageDuplicates, boolean manageFault) { + if (manageFault) { + try { + DatabaseFactory.executeSQLUpdate("drop table " + currentconfig.getParam("DistributionTable"), dbHibConnection); + } catch (Exception e) { + + } + } + } + + public static void main2(String[] args) throws Exception { + String configPath = "./cfg/"; + String csquareTable = "hcaf_d"; + String preprocessedTable = "maxminlat_hspen"; + String envelopeTable = "hspen"; + int numberOfResources = 1; + String userName = "gianpaolo.coro"; + String generatorName = "AQUAMAPS_SUITABLE"; + String scope = "/gcube"; + String finalDistributionTable = "hspec_suitable_executor_all1"; + // String finalDistributionTable = "hspec2012_04_11_18_41_01_575"; + AlgorithmConfiguration config = GeneratorT.getGenerationConfig(numberOfResources, generatorName, envelopeTable, preprocessedTable, "", userName, csquareTable, finalDistributionTable, configPath); + config.setPersistencePath("./"); + config.setGcubeScope(scope); + config.setParam("ServiceUserName", "gianpaolo.coro"); + config.setParam("CreateTable", "false"); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://dbtest.research-infrastructures.eu/aquamapsorgupdated"); + config.setParam("DatabaseDriver", "org.hibernate.dialect.PostgreSQLDialect"); + config.setTableSpace("tbsp_1"); + AquamapsSuitableNode node = new AquamapsSuitableNode(); + node.setup(config); + node.postProcess(true,false); + + } + + // pre process: add default min and max information + public void singleStepPreprocess(Object species, boolean duplicate) throws Exception { + + List speciesObservations = operations.allSpeciesObservations.get(AquamapsSuitableFunctions.getMainInfoID(species)); + + if (((speciesObservations == null) || speciesObservations.size() == 0)) { + Object[] defaultmaxmin = { "90", "-90" }; + speciesObservations = new ArrayList(); + speciesObservations.add(defaultmaxmin); + } + + String speciesID = AquamapsSuitableFunctions.getMainInfoID(species); + // if this is a duplicate message, cancel all associate information to this species + if (duplicate) { + System.out.println("Deleting Duplicates for species " + speciesID); + long t0 = System.currentTimeMillis(); + DatabaseFactory.executeSQLUpdate(String.format(AquamapsSuitableFunctions.deleteDuplicates, currentconfig.getParam("DistributionTable"), speciesID), dbHibConnection); + System.out.println("Duplicates deleted in " + (System.currentTimeMillis() - t0)); + } + + operations.getBoundingBoxInformation((Object[]) species, (Object[]) speciesObservations.get(0)); + } + + // writes the distribution model on the DB: input species vector + list of areas vectors to report + public void singleStepPostprocess(Object species) { + // write info on DB + List toWrite = new ArrayList(); + String speciesID = AquamapsSuitableFunctions.getMainInfoID(species); + Map csquaresMap = operations.completeDistribution.get(speciesID); + if (csquaresMap != null) { + // write only processed areas + for (String singleCsquare : csquaresMap.keySet()) { + + String additionalInformation = operations.getAdditionalInformation(species, operations.processedAreas.get(singleCsquare)); + if (additionalInformation == null) + additionalInformation = ""; + else if (additionalInformation.length() > 0) + additionalInformation = "," + additionalInformation.trim(); + + float prob = 0f; + try { + prob = csquaresMap.get(singleCsquare); + } catch (Exception e) { + System.out.println("Aquamaps Algorithm Single Step PostProcess ->Error in getting probability value at " + speciesID + " , " + singleCsquare); + } + if (prob > 0) + toWrite.add("'" + speciesID + "','" + singleCsquare + "','" + MathFunctions.roundDecimal(prob, 3) + "'" + additionalInformation); + } + + AquamapsSuitableFunctions.writeOnDB(toWrite, currentconfig.getParam("DistributionTable"), dbHibConnection); + } + + } + + @Override + public StatisticalType getOutput() { + List templateHspec = new ArrayList(); + templateHspec.add(TableTemplates.HSPEC); + OutputTable p = new OutputTable(templateHspec,destinationTableLabel,destinationTable,"Output hspec table"); + return p; + } + + + @Override + public int getNumberOfRightElements() { + return operations.numberOfSpecies; + } + + @Override + public int getNumberOfLeftElements() { + return operations.numberOfCells; + } + + + @Override + public void stop() { + DatabaseUtils.closeDBConnection(dbHibConnection); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableNode1v.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableNode1v.java new file mode 100644 index 0000000..6caf6e9 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/AquamapsSuitableNode1v.java @@ -0,0 +1,566 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Timer; +import java.util.TimerTask; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.SpatialProbabilityDistributionNode; +import org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsAlgorithmCore; +import org.gcube.dataanalysis.ecoengine.spatialdistributions.AquamapsSuitable; +import org.gcube.dataanalysis.ecoengine.user.GeneratorT; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.gcube.dataanalysis.ecoengine.utils.Operations; +import org.hibernate.SessionFactory; + +public class AquamapsSuitableNode1v extends SpatialProbabilityDistributionNode { + + // variables + protected String countSpeciesQuery, countCellsQuery, createTableStatement, destinationTable, destinationTableLabel, type; + protected AquamapsAlgorithmCore core; + protected AlgorithmConfiguration currentconfig; + protected AquamapsSuitableFunctions operations; + protected SessionFactory dbHibConnection; + + public AquamapsSuitableNode1v() { + + } + + // Processing + // an initialization phase in which the input are initialized and then dumped to files + @Override + public void setup(AlgorithmConfiguration config) throws Exception { + // init algorithm + long t00 = System.currentTimeMillis(); + initSingleNode(config); + long t0 = System.currentTimeMillis(); + String distributionTable = config.getParam("DistributionTable"); + // System.out.println("Dumping information .. in " + config.getPersistencePath() + config.getModel() + "/"); + // operations.dumpAll(config.getPersistencePath() + config.getModel() + "/"); + // System.out.println(".. Dump Done in " + (System.currentTimeMillis() - t0) + " ms"); + // create table + boolean createTable = config.getParam("CreateTable") != null ? Boolean.parseBoolean(config.getParam("CreateTable")) : false; + System.out.println("Creating table " + distributionTable + " " + createTable); + if (createTable) { + try { + DatabaseUtils.dropTableStatement(distributionTable); + } catch (Exception e) { + } + DatabaseUtils.createBigTable(createTable, distributionTable, config.getDatabaseDriver(), config.getDatabaseUserName(), config.getDatabasePassword(), config.getDatabaseURL(), createTableStatement, dbHibConnection); + } + //create minmaxlat table if necessary + createMaxMinHspen(config); + // close connection + // DatabaseUtils.closeDBConnection(dbHibConnection); + System.out.println("Preparation phase finished in " + (System.currentTimeMillis() - t00) + " ms"); + + /* + * databasecheckScheduler = new Timer(); databasecheckScheduler.schedule(new DatabaseController(), 0, refreshTime); + */ + } + + + private void createMaxMinHspen(AlgorithmConfiguration config){ + + String hspenMinMaxLat=""; + if ((config.getParam("PreprocessedTable")!=null)&&(config.getParam("PreprocessedTable").length()>0)) + hspenMinMaxLat = config.getParam("PreprocessedTable"); + + //if not preprocessed then generate a preprocessed table + if ((hspenMinMaxLat==null)||(hspenMinMaxLat.length()==0)){ + //take the name of the hspen table + String hspenTable = config.getParam("EnvelopeTable"); + //check if the table exists + String supposedminmaxlattable = "maxminlat_"+hspenTable; + List select = null; + try{ + select = DatabaseFactory.executeSQLQuery("select * from "+supposedminmaxlattable+" limit 1",dbHibConnection); + }catch(Exception ee){} + //if it exists then set the table name + if (select!=null){ + hspenMinMaxLat = supposedminmaxlattable; + System.out.println("Aquamaps Algorithm Node Init ->the min max latitudes table yet exists "+hspenMinMaxLat); + } + else{ + //otherwise create it by calling the creator + System.out.println("Aquamaps Algorithm Node Init ->the min max latitudes table does not exist! - generating"); + hspenMinMaxLat = AquamapsSuitable.generateMaxMinHspec(supposedminmaxlattable, hspenTable,config.getParam("OccurrencePointsTable"), dbHibConnection); + System.out.println("Aquamaps Algorithm Node Init ->min max latitudes table created in "+hspenMinMaxLat); + } + } + config.setParam("PreprocessedTable",hspenMinMaxLat); + } + + // initializations + public void initSingleNode(AlgorithmConfiguration config) { + initCore(config); + // take all species records + System.out.println("Aquamaps Algorithm Init ->Taking all species records"); + operations.numberOfSpecies = Integer.parseInt("" + (DatabaseFactory.executeSQLQuery(countSpeciesQuery, dbHibConnection)).get(0)); + // take all csquares records + System.out.println("Aquamaps Algorithm Init ->Count all csquares"); + operations.numberOfCells = Integer.parseInt("" + DatabaseFactory.executeSQLQuery(countCellsQuery, dbHibConnection).get(0)); + // take all csquares records + /* + * System.out.println("Aquamaps Algorithm ->Taking csquares"); String CellsQuery = String.format(AquamapsSuitableFunctions.csquareCodeQuery, currentconfig.getParam("CsquarecodesTable"), "" + operations.numberOfCells, "0"); operations.environmentVectors = DatabaseFactory.executeSQLQuery(CellsQuery, dbHibConnection); + */ + System.out.println("Aquamaps Algorithm Init ->init finished"); + } + + private void initCore(AlgorithmConfiguration config) { + currentconfig = config; + System.out.println("Aquamaps Algorithm-> Initializing DB connection"); + dbHibConnection = DatabaseUtils.initDBSession(config); + System.out.println("Aquamaps Algorithm-> Initializing DB connection OK"); + + countSpeciesQuery = String.format(AquamapsSuitableFunctions.countAllSpeciesQuery, config.getParam("EnvelopeTable")); + countCellsQuery = String.format(AquamapsSuitableFunctions.countCsquareCodeQuery, config.getParam("CsquarecodesTable")); + + createTableStatement = String.format(AquamapsSuitableFunctions.createTableStatement, config.getParam("DistributionTable")); + if (config.getTableSpace() != null) { + createTableStatement = createTableStatement.replace("#TABLESPACE#", "TABLESPACE " + config.getTableSpace()); + } else + createTableStatement = createTableStatement.replace("#TABLESPACE#", ""); + + destinationTable = config.getParam("DistributionTable"); + destinationTableLabel = config.getParam("DistributionTableLabel"); + System.out.println("Aquamaps Algorithm-> Initializing Core Algorithm"); + core = new AquamapsAlgorithmCore(); + operations = new AquamapsSuitableFunctions(core, type, config); + System.out.println("Aquamaps Algorithm-> Core Initializiation OK "); + } + + // end initialization + + // the core execution of a node + @Override + public int executeNode(int cellOrdinal, int chunksize, int speciesOrdinal, int speciesChunkSize, boolean duplicate, String pathToFiles, String nodeConfigurationFileObject, String logfile) { + long t00 = System.currentTimeMillis(); + try { + System.out.println("Aquamaps Suitable: " + cellOrdinal + " Node Started"); + // rebuild all files + core = new AquamapsAlgorithmCore(); + operations = new AquamapsSuitableFunctions(new AquamapsAlgorithmCore(), type, null); + System.out.println("Aquamaps Suitable: " + cellOrdinal + " Rebuilding objects"); + long t0 = System.currentTimeMillis(); + operations.rebuildConfig(new File(pathToFiles, nodeConfigurationFileObject).getAbsolutePath()); + // operations.rebuildAll(cellOrdinal, chunksize, speciesOrdinal, speciesChunkSize, pathToFiles); + operations.currentconfig.setConfigPath(pathToFiles); + currentconfig = operations.currentconfig; + System.out.println("Aquamaps Suitable: " + speciesOrdinal + " Objects Rebuilt in " + (System.currentTimeMillis() - t0) + " ms"); + String SpeciesQuery = String.format(AquamapsSuitableFunctions.selectAllSpeciesQuery, currentconfig.getParam("EnvelopeTable"), "" + speciesChunkSize, "" + speciesOrdinal); + String CellsQuery = String.format(AquamapsSuitableFunctions.csquareCodeQuery, currentconfig.getParam("CsquarecodesTable"), "" + chunksize, "" + cellOrdinal); + System.out.println("Aquamaps Algorithm-> Initializing DB connection with parameters: " + currentconfig.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile + " " + currentconfig.getParam("DatabaseDriver") + " " + currentconfig.getParam("DatabaseUserName") + " " + currentconfig.getParam("DatabasePassword") + " " + currentconfig.getParam("DatabaseURL")); + System.out.println("Aquamaps Algorithm-> File Exists: " + new File(currentconfig.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile).exists()); + System.out.println("Aquamaps Algorithm-> File IS READABLE: " + new File(currentconfig.getConfigPath() + AlgorithmConfiguration.defaultConnectionFile).canRead()); + dbHibConnection = DatabaseUtils.initDBSession(currentconfig); + + System.out.println("Aquamaps Algorithm-> Building Species Observations Max-Min Latitutes"); + operations.allSpeciesObservations = new HashMap>(); + String hspenMinMaxLat = currentconfig.getParam("PreprocessedTable"); + System.out.println("Aquamaps Algorithm-> Preprocessed Table is " + hspenMinMaxLat); + System.out.println("Aquamaps Algorithm-> Query: " + String.format(AquamapsSuitableFunctions.selectAllSpeciesObservationQuery, hspenMinMaxLat)); + System.out.println("Aquamaps Algorithm-> DB CONNECTION " + dbHibConnection); + List SpeciesObservations = DatabaseFactory.executeSQLQuery(String.format(AquamapsSuitableFunctions.selectAllSpeciesObservationQuery, hspenMinMaxLat), dbHibConnection); + System.out.println("Aquamaps Algorithm-> Queried"); + if (SpeciesObservations == null) + SpeciesObservations = new ArrayList(); + System.out.println("Aquamaps Algorithm-> Obtained Observations " + SpeciesObservations.size()); + + int lenObservations = SpeciesObservations.size(); + for (int i = 0; i < lenObservations; i++) { + Object[] maxminArray = (Object[]) SpeciesObservations.get(i); + String speciesid = (String) maxminArray[0]; + List maxminInfo = new ArrayList(); + maxminInfo.add(maxminArray); + operations.allSpeciesObservations.put((String) speciesid, maxminInfo); + } + + // take all species records + System.out.println("Aquamaps Algorithm ->Taking species records"); + operations.speciesVectors = DatabaseFactory.executeSQLQuery(SpeciesQuery, dbHibConnection); + // take all csquares records + System.out.println("Aquamaps Algorithm ->Taking csquares"); + if (operations.environmentVectors == null) + operations.environmentVectors = DatabaseFactory.executeSQLQuery(CellsQuery, dbHibConnection); + + int maxSpecies = operations.speciesVectors.size(); + int maxCells = operations.environmentVectors.size(); + + System.out.println("Aquamaps Algorithm ->Processing " + maxCells + " cells VS " + maxSpecies + " species"); + + System.out.println("Aquamaps Algorithm ->Focus On Table: " + currentconfig.getParam("DistributionTable")); + for (int j = 0; j < maxSpecies; j++) { + System.out.println("Aquamaps Suitable: calculating elements from " + cellOrdinal + " to " + maxCells + " for species " + j); + Object speciesV = operations.speciesVectors.get(j); + System.out.println("Aquamaps Suitable: " + cellOrdinal + " calculating probabilities"); + singleStepPreprocess(speciesV, duplicate); + for (int i = 0; i < maxCells; i++) { + Object area = operations.environmentVectors.get(i); + operations.calcProb(speciesV, area); + } + + System.out.println("Aquamaps Suitable: " + cellOrdinal + " probabilities calculated!"); + System.out.println("Aquamaps Suitable: " + cellOrdinal + " writing on DB"); + singleStepPostprocess(speciesV); + System.out.println("Aquamaps Suitable: " + cellOrdinal + " write on DB OK"); + } + + + + try { + FileWriter fw = new FileWriter(new File(logfile)); + fw.write("OK"); + fw.close(); + } catch (Exception e) { + + } + } catch (Exception e) { + System.err.println("ERROR " + e); + e.printStackTrace(); + } + finally{ + + DatabaseUtils.closeDBConnection(dbHibConnection); + System.out.println("Aquamaps Suitable: " + cellOrdinal + " Database Closed!"); + + } + System.out.println("Aquamaps Suitable: " + cellOrdinal + " Node Ended in " + ((float) (System.currentTimeMillis() - t00) / 1000f) + " s"); + return 0; + } + + // Auxiliary Functions + // to overwrite in case of native generation in order to filer on the probabilities types + @Override + public List getInputParameters() { + List parameters = new AquamapsSuitable().getInputParameters(); + return parameters; + } + + public Queue filterProbabilitySet(Queue probabiltyRows) { + return probabiltyRows; + } + + public float getInternalStatus() { + return 100; + } + + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PARALLEL_SPECIES_VS_CSQUARE_FROM_DATABASE}; + return p; + } + + public String getName() { + return "AQUAMAPS_SUITABLE"; + } + + public String getDescription() { + return "Algorithm by Aquamaps on a single node"; + } + + public void postProcess(boolean manageDuplicates, boolean manageFault) { + if (databasecheckScheduler != null) + databasecheckScheduler.cancel(); + if (manageFault) { + try { + DatabaseFactory.executeSQLUpdate("drop table " + currentconfig.getParam("DistributionTable"), dbHibConnection); + } catch (Exception e) { + + } + } + } + + public void postProcess4(boolean manageDuplicates) { + if (databasecheckScheduler != null) + databasecheckScheduler.cancel(); + + if (manageDuplicates) { + try { + + long t0 = System.currentTimeMillis(); + System.out.println("Deleting duplicates ... this can require very much time"); + String distributionTable = currentconfig.getParam("DistributionTable"); + String countQuery = String.format("select count(*) from %1$s ", distributionTable); + + System.out.println("Taking number of elements ... "); + long t00 = System.currentTimeMillis(); + /* + * List explain = DatabaseFactory.executeSQLQuery(String.format(AquamapsSuitableFunctions.countAll, currentconfig.getParam("DistributionTable")), dbHibConnection); String explained = "" + explain.get(0); explained = explained.substring(explained.lastIndexOf("rows=")); explained = explained.substring(explained.indexOf('=') + 1, explained.indexOf(' ')); int nElements = (int )(Integer.parseInt(explained)*1.5); + */ + int nElements = Integer.parseInt("" + (DatabaseFactory.executeSQLQuery(countQuery, dbHibConnection)).get(0)); + System.out.println("Calculated " + nElements + " elements in " + (System.currentTimeMillis() - t00)); + int maxRounds = 100; + int[] chunkSizes = Operations.takeChunks(nElements, maxRounds); + // String selectChunk = "select speciesid,csquarecode, '' || ctid as ctidstring from %1$s limit %2$s offset %3$s"; + String selectChunk = "select speciesid,csquarecode, CAST( ctid AS text ) from %1$s limit %2$s offset %3$s"; + int offset = 0; + HashMap mapSpecies = new HashMap(); + List duplicates = new ArrayList(); + t00 = System.currentTimeMillis(); + for (int i = 0; i < maxRounds; i++) { + // for (int i = 0; i < 5; i++) { + String query = String.format(selectChunk, distributionTable, "" + chunkSizes[i], "" + offset); + offset += chunkSizes[i]; + System.out.println("Selecting " + chunkSizes[i] + " from " + offset + " : " + query); + long t01 = System.currentTimeMillis(); + List objs = DatabaseFactory.executeSQLQuery(query, dbHibConnection); + System.out.println("Selected in " + (System.currentTimeMillis() - t01)); + for (Object rowO : objs) { + Object[] row = (Object[]) rowO; + String species = "" + row[0]; + String csquareCode = "" + row[1]; + String ctid = "" + row[2]; + String key = species + ":" + csquareCode; + if (mapSpecies.containsKey(key)) { + duplicates.add(ctid); + } else + mapSpecies.put(key, ctid); + } + System.out.println("Status " + ((float) i / (float) maxRounds) * 100f); + } + System.out.println("Finished map calculation in " + (System.currentTimeMillis() - t00)); + + t00 = System.currentTimeMillis(); + StringBuffer toDelete = new StringBuffer(); + toDelete.append("delete * from %1$s where ctid in ("); + int size = duplicates.size(); + if (size > 0) { + System.out.println("Building Deleting Statement ..."); + for (int j = 0; j < size; j++) { + toDelete.append(duplicates.get(j)); + if (j != size - 1) { + toDelete.append(","); + } + } + + toDelete.append(")"); + System.out.println("Finished Building Deleting Statement in " + (System.currentTimeMillis() - t00)); + + System.out.println("Deleting ..."); + t00 = System.currentTimeMillis(); + DatabaseFactory.executeSQLUpdate(toDelete.toString(), dbHibConnection); + System.out.println("... Deleted in " + (System.currentTimeMillis() - t00)); + } else + System.out.println("No Duplications to delete"); + + System.out.println("Deleted All duplicates in " + (System.currentTimeMillis() - t0)); + } catch (Exception e) { + System.out.println("An Error Occurred"); + e.printStackTrace(); + } + + } + + } + + public void postProcess1(boolean manageDuplicates) { + if (databasecheckScheduler != null) + databasecheckScheduler.cancel(); + + if (manageDuplicates) { + long t0 = System.currentTimeMillis(); + System.out.println("Deleting duplicates ... this can require very much time"); + String distributionTable = currentconfig.getParam("DistributionTable"); + // String deletecommand = String.format("delete from %1$s where exists ( select * from %1$s i where i.speciesid = %1$s.speciesid and i.csquarecode = %1$s.csquarecode and i.ctid < %1$s.ctid)", currentconfig.getParam("DistributionTable")); + String deletecommand = String.format("select * into testtable from %1$s where exists (select * from %1$s i where i.speciesid = %1$s.speciesid and i.csquarecode = %1$s.csquarecode and i.ctid < %1$s.ctid)", currentconfig.getParam("DistributionTable")); + // DELETE FROM %1$s WHERE ctid NOT IN(SELECT MAX(s.ctid) FROM %1$s s GROUP BY (s.speciesid,s.csquarecode))", + + /* + * String countQuery = String.format("select count(*) from %1$s ",distributionTable); int nElements = Integer.parseInt(""+(DatabaseFactory.executeSQLQuery(countQuery, dbHibConnection)).get(0)); int maxRounds = 100; + * + * int[] chunkSizes = Operations.takeChunks(nElements, 100); String selectChunk = "select speciesid,csquarecode,ctid from %1$s limit %2$s offset %3$s"; int offset = 0; HashMap mapSpecies = new HashMap(); List duplicates = new ArrayList(); for (int i=0;i objs = DatabaseFactory.executeSQLQuery(query, dbHibConnection); for (Object rowO:objs) { Object[] row = (Object[]) rowO; String species = ""+row[0]; String csquareCode = ""+row[1]; String ctid = ""+row[3]; String key = species+":"+csquareCode; if (mapSpecies.containsKey(key)){ duplicates.add(ctid); } else mapSpecies.put(key, ctid); } } + * + * StringBuffer toDelete = new StringBuffer(); toDelete.append("delete * from %1$s where ctid in ("); int size = duplicates.size(); for (int j=0;j speciesObservations = operations.allSpeciesObservations.get(AquamapsSuitableFunctions.getMainInfoID(species)); + + if (((speciesObservations == null) || speciesObservations.size() == 0)) { + Object[] defaultmaxmin = { "90", "-90" }; + speciesObservations = new ArrayList(); + speciesObservations.add(defaultmaxmin); + } + + String speciesID = AquamapsSuitableFunctions.getMainInfoID(species); + // if this is a duplicate message, cancel all associate information to this species + if (duplicate) { + System.out.println("Deleting Duplicates for species " + speciesID); + long t0 = System.currentTimeMillis(); + DatabaseFactory.executeSQLUpdate(String.format(AquamapsSuitableFunctions.deleteDuplicates, currentconfig.getParam("DistributionTable"), speciesID), dbHibConnection); + System.out.println("Duplicates deleted in " + (System.currentTimeMillis() - t0)); + } + + operations.getBoundingBoxInformation((Object[]) species, (Object[]) speciesObservations.get(0)); + } + + // writes the distribution model on the DB: input species vector + list of areas vectors to report + public void singleStepPostprocess(Object species) { + // write info on DB + List toWrite = new ArrayList(); + String speciesID = AquamapsSuitableFunctions.getMainInfoID(species); + Map csquaresMap = operations.completeDistribution.get(speciesID); + if (csquaresMap != null) { + // write only processed areas + for (String singleCsquare : csquaresMap.keySet()) { + + String additionalInformation = operations.getAdditionalInformation(species, operations.processedAreas.get(singleCsquare)); + if (additionalInformation == null) + additionalInformation = ""; + else if (additionalInformation.length() > 0) + additionalInformation = "," + additionalInformation.trim(); + + float prob = 0f; + try { + prob = csquaresMap.get(singleCsquare); + } catch (Exception e) { + System.out.println("Aquamaps Algorithm Single Step PostProcess ->Error in getting probability value at " + speciesID + " , " + singleCsquare); + } + if (prob > 0) + toWrite.add("'" + speciesID + "','" + singleCsquare + "','" + MathFunctions.roundDecimal(prob, 2) + "'" + additionalInformation); + } + + AquamapsSuitableFunctions.writeOnDB(toWrite, currentconfig.getParam("DistributionTable"), dbHibConnection); + } + + } + + @Override + public StatisticalType getOutput() { + List templateHspec = new ArrayList(); + templateHspec.add(TableTemplates.HSPEC); + OutputTable p = new OutputTable(templateHspec,destinationTableLabel,destinationTable,"Output hspec table"); + return p; + } + + + @Override + public int getNumberOfSpecies() { + return operations.numberOfSpecies; + } + + @Override + public int getNumberOfGeoInfo() { + return operations.numberOfCells; + } + + private int lastProcessed; + Timer databasecheckScheduler; + private static int refreshTime = 10000; + + @Override + public int getNumberOfProcessedSpecies() { + return lastProcessed; + } + + private class DatabaseController extends TimerTask { + + @Override + public void run() { + try { + List explain = DatabaseFactory.executeSQLQuery(String.format(AquamapsSuitableFunctions.countAll, currentconfig.getParam("DistributionTable")), dbHibConnection); + String explained = "" + explain.get(0); + explained = explained.substring(explained.lastIndexOf("rows=")); + explained = explained.substring(explained.indexOf('=') + 1, explained.indexOf(' ')); + AnalysisLogger.getLogger().debug("ESTIMATED ROWS: " + explained); + lastProcessed = Integer.parseInt(explained); + } catch (Exception e) { + try { + e.printStackTrace(); + /* + * List all = DatabaseFactory.executeSQLQuery(String.format(AquamapsSuitableFunctions.countAll, currentconfig.getParam("DistributionTable")), dbHibConnection); int numberOfElements = Integer.parseInt("" + all.get(0)); lastProcessed = numberOfElements; + */ + } catch (Exception e2) { + e.printStackTrace(); + } + } + } + } + + @Override + public void stop() { + try{ + databasecheckScheduler.cancel(); + }catch(Exception e){} + DatabaseUtils.closeDBConnection(dbHibConnection); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/CMSY.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/CMSY.java new file mode 100644 index 0000000..28c9024 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/CMSY.java @@ -0,0 +1,152 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.utils.IOHelper; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.executor.scripts.OSCommand; +import org.gcube.dataanalysis.executor.util.RScriptsManager; +import org.gcube.dataanalysis.executor.util.StorageUtils; + +public class CMSY extends ActorNode { + + public int count; + + public float status = 0; + + @Override + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON }; + return p; + } + + @Override + public String getName() { + return "CMSY"; + } + + @Override + public String getDescription() { + return "An algorithm to estimate the Maximum Sustainable Yield from a catch statistic. If also a Biomass trend is provided, MSY estimation is provided also with higher precision. The method has been developed by R. Froese, G. Coro, N. Demirel and K. Kleisner."; + } + + static String idsFile = "IDsFile"; + static String stocksFile = "StocksFile"; + static String stock= "SelectedStock"; + static String processOutput= "ProcessOutput"; + static String scriptName = "CMSY_22_noplot.R"; + + @Override + public List getInputParameters() { + + List parameters = new ArrayList(); + + IOHelper.addStringInput(parameters, idsFile, "Http link to a file containing prior information about the stocks, in WKLife IV format. Example: http://goo.gl/9rg3qK", ""); + IOHelper.addStringInput(parameters, stocksFile, "Http link to a file containing catch and biomass (or CPUE) trends , in WKLife IV format. Example: http://goo.gl/Mp2ZLY", ""); + IOHelper.addStringInput(parameters,stock,"The stock on which the procedure has to focus e.g. HLH_M07","" ); + + return parameters; + } + + @Override + public StatisticalType getOutput() { + File outfile = new File(config.getPersistencePath(),config.getParam(processOutput)); + PrimitiveType o = new PrimitiveType(File.class.getName(), outfile, PrimitiveTypes.FILE, "OutputFile", "Output file"); + return o; + } + + @Override + public void initSingleNode(AlgorithmConfiguration config) { + + } + + @Override + public float getInternalStatus() { + return status; + } + + String outputFileName; + AlgorithmConfiguration config; + + @Override + public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStartIndex, int numberOfRightElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) { + try { + status = 0; + config = Transformations.restoreConfig(nodeConfigurationFileObject); + String outputFile = config.getParam(processOutput); + AnalysisLogger.getLogger().info("CMSY expected output "+outputFile); + + String fileid=new File(sandboxFolder,"WKLIFE4ID.csv").getAbsolutePath(); + String filestock=new File(sandboxFolder,"WKLIFE4Stocks.csv").getAbsolutePath(); + StorageUtils.downloadInputFile(config.getParam(idsFile), fileid); + StorageUtils.downloadInputFile(config.getParam(stocksFile), filestock); + AnalysisLogger.getLogger().debug("Check fileID: "+fileid+" "+new File(fileid).exists()); + AnalysisLogger.getLogger().debug("Check fileStocks: "+filestock+" "+new File(filestock).exists()); + RScriptsManager scriptmanager = new RScriptsManager(); + + HashMap codeinj = new HashMap(); + codeinj.put("HLH_M07",config.getParam(stock)); + config.setConfigPath("./"); + scriptmanager.executeRScript(config, scriptName, "", new HashMap(), "", "outputfile.txt", codeinj, true,false,false); + + outputFileName = scriptmanager.getCurrentOutputFileName(); + String outputFilePath = new File(sandboxFolder,outputFile).getAbsolutePath(); + AnalysisLogger.getLogger().info("CMSY writing output file in path "+outputFilePath); + OSCommand.FileCopy(outputFileName,outputFilePath); + AnalysisLogger.getLogger().info("CMSY uploading output file "+outputFile); + StorageUtils.uploadFilesOnStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), sandboxFolder,outputFile); + AnalysisLogger.getLogger().info("CMSY Finished"); + }catch(Exception e){ + e.printStackTrace(); + } + + return 0; + } + + @Override + public void setup(AlgorithmConfiguration config) throws Exception { + this.config = config; + AnalysisLogger.getLogger().info("CMSY process is initialized"); + config.setParam(processOutput, "CMSY_"+"output_"+(UUID.randomUUID()+".txt").replace("-", "")); + } + + @Override + public int getNumberOfRightElements() { + return 1; + } + + @Override + public int getNumberOfLeftElements() { + return 1; + } + + @Override + public void stop() { + AnalysisLogger.getLogger().info("CMSY process stopped"); + } + + boolean haspostprocessed = false; + + @Override + public void postProcess(boolean manageDuplicates, boolean manageFault) { + try { + String filename = config.getParam(processOutput); + StorageUtils.downloadFilefromStorage(config.getGcubeScope(), config.getParam("ServiceUserName"), config.getPersistencePath(), filename); + AnalysisLogger.getLogger().debug("CMSY - Got file from Storage: "+filename); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/LWR.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/LWR.java new file mode 100644 index 0000000..c8ceec9 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/algorithms/LWR.java @@ -0,0 +1,377 @@ +package org.gcube.dataanalysis.executor.nodes.algorithms; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.List; + +import org.gcube.common.core.utils.logging.GCUBELog; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.job.management.DistributedProcessingAgent; +import org.gcube.dataanalysis.executor.job.management.QueueJobManager; +import org.gcube.dataanalysis.executor.scripts.OSCommand; +import org.hibernate.SessionFactory; + +public class LWR extends ActorNode { + + public String destinationTable; + public String destinationTableLabel; + public String originTable; + public String familyColumn; + public int count; + + public float status = 0; + public int prevbroadcastTimePeriod; + public int prevmaxNumberOfStages; + public int prevmaxMessages; + + private SessionFactory dbconnection; + private static String createOutputTable = "CREATE TABLE %1$s (Fam character varying(255), SF character varying(255), BS character varying(255), SpC character varying(255), LWR real, priormeanlog10a real, priorsdlog10a real, priormeanb real, priorsdb real, note character varying(255) )"; + private static String columnNames = "Fam,SF,BS,SpC,LWR,priormeanlog10a,priorsdlog10a,priormeanb,priorsdb,note"; + + @Override + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON }; + return p; + } + + @Override + public String getName() { + return "LWR"; + } + + @Override + public String getDescription() { + return "An algorithm to estimate Length-Weight relationship parameters for marine species, using Bayesian methods. Runs an R procedure. Based on the Cube-law theory."; + } + + @Override + public List getInputParameters() { + List templateLWRInput = new ArrayList(); + templateLWRInput.add(TableTemplates.GENERIC); + InputTable p1 = new InputTable(templateLWRInput, "LWR_Input", "Input table containing taxa and species information", "lwr"); + ColumnType p3 = new ColumnType("LWR_Input", "FamilyColumn", "The column containing Family information", "Family", false); + ServiceType p4 = new ServiceType(ServiceParameters.RANDOMSTRING, "RealOutputTable", "name of the resulting table", "lwr_"); + PrimitiveType p2 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, "TableLabel", "Name of the table which will contain the model output", "lwrout"); + + List parameters = new ArrayList(); + parameters.add(p1); + parameters.add(p3); + parameters.add(p2); + parameters.add(p4); + + DatabaseType.addDefaultDBPars(parameters); + + return parameters; + } + + @Override + public StatisticalType getOutput() { + List template = new ArrayList(); + template.add(TableTemplates.GENERIC); + OutputTable p = new OutputTable(template, destinationTableLabel, destinationTable, "Output lwr table"); + return p; + } + + @Override + public void initSingleNode(AlgorithmConfiguration config) { + + } + + @Override + public float getInternalStatus() { + return status; + } + + private static String scriptName = "UpdateLWR_4.R"; +// private static String scriptName = "UpdateLWR_Test2.R"; + + @Override + public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStartIndex, int numberOfRightElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) { + String insertQuery = null; + try { + status = 0; + AlgorithmConfiguration config = Transformations.restoreConfig(nodeConfigurationFileObject); + config.setConfigPath(sandboxFolder); + System.out.println("Initializing DB"); + dbconnection = DatabaseUtils.initDBSession(config); + destinationTableLabel = config.getParam("TableLabel"); + destinationTable = config.getParam("RealOutputTable"); + + System.out.println("Destination Table: "+destinationTable); + System.out.println("Destination Table Label: "+destinationTableLabel); + + + originTable = config.getParam("LWR_Input"); + familyColumn = config.getParam("FamilyColumn"); + System.out.println("Origin Table: "+originTable); + + // take the families to process + List families = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(originTable, familyColumn, ""), dbconnection); + + // transform the families into a string + // Families <- Fam.All[Fam.All== "Acanthuridae" | Fam.All == "Achiridae"] + StringBuffer familiesFilter = new StringBuffer(); + familiesFilter.append("Families <- Fam.All["); + + /* + rightStartIndex=519; + numberOfRightElementsToProcess=19; + */ + + int end = rightStartIndex + numberOfRightElementsToProcess; + + for (int i = rightStartIndex; i < end; i++) { + familiesFilter.append("Fam.All == \"" + families.get(i) + "\""); + if (i < end - 1) + familiesFilter.append(" | "); + } + familiesFilter.append("]"); + + OSCommand.ExecuteGetLine("ls", null); + OSCommand.ExecuteGetLine("pwd", null); + OSCommand.ExecuteGetLine("chmod +x * | whoami", null); + + String substitutioncommand = "sed -i 's/Families <- Fam.All[Fam.All== \"Acanthuridae\" | Fam.All == \"Achiridae\"]/" + familiesFilter + "/g' " + "UpdateLWR_Test2.R"; +// System.out.println("Executing command: " + substitutioncommand); + System.out.println("Preparing for processing the families names: "+familiesFilter.toString()); + + // substitute the string in the RCode +// OSCommand.ExecuteGetLine(substitutioncommand, null); +// substring(sandboxFolder+"UpdateLWR_Test2.R",sandboxFolder+"UpdateLWR_Tester.R","Families <- Fam.All[Fam.All== \"Acanthuridae\" | Fam.All == \"Achiridae\"]",familiesFilter.toString()); + + substring(sandboxFolder+scriptName,sandboxFolder+"UpdateLWR_Tester.R","Families <- Fam.All[Fam.All== \"Acanthuridae\" | Fam.All == \"Achiridae\"]",familiesFilter.toString()); + //for test only +// substring(sandboxFolder+scriptName,sandboxFolder+"UpdateLWR_Tester.R","Families <- Fam.All[Fam.All== \"Acanthuridae\" | Fam.All == \"Achiridae\"]","Families <- Fam.All[Fam.All== \"Abyssocottidae\"]"); + +// substring(sandboxFolder+scriptName,sandboxFolder+"UpdateLWR_Tester.R","Families <- Fam.All[Fam.All== \"Acanthuridae\" | Fam.All == \"Achiridae\"]","Families <- Fam.All[Fam.All== \"Sparidae\"]"); + + System.out.println("Creating local file from remote table"); + // download the table in csv + DatabaseUtils.createLocalFileFromRemoteTable(sandboxFolder+"RF_LWR.csv", originTable, ",", config.getDatabaseUserName(),config.getDatabasePassword(),config.getDatabaseURL()); + + String headers = "Subfamily,Family,Genus,Species,FBname,SpecCode,AutoCtr,Type,a,b,CoeffDetermination,Number,LengthMin,Score,BodyShapeI"; + System.out.println("Adding headers to the file"); + + String headerscommand = "sed -i '1s/^/"+headers+"\\n/g' "+"RF_LWR2.csv"; + // substitute the string in the RCode +// OSCommand.ExecuteGetLine(headerscommand, null); + addheader(sandboxFolder+"RF_LWR.csv",sandboxFolder+"RF_LWR2.csv",headers); + +// OSCommand.ExecuteGetLine("head RF_LWR2.csv", null); + System.out.println("Headers added"); + + System.out.println("Executing R script " + "R --no-save < UpdateLWR_Tester.R"); + + // run the R code + Process process = Runtime.getRuntime().exec("R --no-save"); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(process.getOutputStream())); + bw.write("source('UpdateLWR_Tester.R')\n"); + bw.write("q()\n"); + bw.close(); + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = br.readLine(); + System.out.println(line); + while (line!=null){ + line = br.readLine(); + System.out.println(line); + } + + process.destroy(); + +// OSCommand.ExecuteGetLine("R --no-save lines = new ArrayList(); + BufferedReader br = new BufferedReader(new FileReader(csvfile)); + String line = br.readLine(); + while (line!=null){ + lines.add(line); + line = br.readLine(); + } + br.close(); + + StringBuffer sb = new StringBuffer(); + sb.append("("); + int m = lines.size(); + for (int i=0;i=0){ + line = line.replace(s, sub); + } + bw.write(line+"\n"); + line = br.readLine(); + } + + br.close(); + bw.close(); + } + + + private static void addheader(String file, String newFile, String header) throws Exception{ + BufferedReader br = new BufferedReader(new FileReader(new File(file))); + BufferedWriter bw = new BufferedWriter(new FileWriter(new File(newFile))); + bw.write(header+"\n"); + + String line = br.readLine(); + + while (line!=null){ + bw.write(line+"\n"); + line = br.readLine(); + } + + br.close(); + bw.close(); + } + + @Override + public void setup(AlgorithmConfiguration config) throws Exception { + + destinationTableLabel = config.getParam("TableLabel"); + AnalysisLogger.getLogger().info("Table Label: "+destinationTableLabel); + destinationTable = config.getParam("RealOutputTable"); + AnalysisLogger.getLogger().info("Uderlying Table Name: "+destinationTable); + originTable = config.getParam("LWR_Input"); + AnalysisLogger.getLogger().info("Original Table: "+originTable); + familyColumn = config.getParam("FamilyColumn"); + AnalysisLogger.getLogger().info("Family Column: "+familyColumn); + haspostprocessed = false; + + AnalysisLogger.getLogger().info("Initializing DB Connection"); + dbconnection = DatabaseUtils.initDBSession(config); + List families = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(originTable, familyColumn, ""), dbconnection); + count = families.size(); +// count = 4; + +// count = 200; + + prevmaxMessages=D4ScienceDistributedProcessing.maxMessagesAllowedPerJob; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=1; + prevbroadcastTimePeriod = QueueJobManager.broadcastTimePeriod; + QueueJobManager.broadcastTimePeriod=4*3600000; + prevmaxNumberOfStages = QueueJobManager.maxNumberOfStages; + QueueJobManager.maxNumberOfStages=10000; + + AnalysisLogger.getLogger().info("Creating Destination Table " + destinationTable); + try{ + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(destinationTable), dbconnection); + }catch (Exception e) { + AnalysisLogger.getLogger().info("Table "+destinationTable+" did not exist"); + } + DatabaseFactory.executeSQLUpdate(String.format(createOutputTable, destinationTable), dbconnection); + AnalysisLogger.getLogger().info("Destination Table Created! Addressing " + count + " species"); + + + } + + @Override + public int getNumberOfRightElements() { + return count; + } + + @Override + public int getNumberOfLeftElements() { + return 1; + } + + @Override + public void stop() { + + //if has not postprocessed, then abort the computations by removing the database table + if (!haspostprocessed){ + try{ + AnalysisLogger.getLogger().info("The procedure did NOT correctly postprocessed ....Removing Table "+destinationTable+" because of computation stop!"); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(destinationTable), dbconnection); + }catch (Exception e) { + AnalysisLogger.getLogger().info("Table "+destinationTable+" did not exist"); + } + } + else + AnalysisLogger.getLogger().info("The procedure has correctly postprocessed: shutting down the connection!"); + if (dbconnection != null) + try { + dbconnection.close(); + } catch (Exception e) { + } + } + + boolean haspostprocessed = false; + @Override + public void postProcess(boolean manageDuplicates, boolean manageFault) { +// D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=previousnumberofspeciesperjob; + QueueJobManager.broadcastTimePeriod=prevbroadcastTimePeriod; + QueueJobManager.maxNumberOfStages=prevmaxNumberOfStages; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=prevmaxMessages; +// select fam, sf, bs, spc , lwr, priormeanlog10a, priorsdlog10a, priormeanb, priorsdb,note into testfam from (select distinct(spc) as a, fam, sf, bs, spc , lwr, priormeanlog10a, priorsdlog10a, priormeanb, priorsdb,note from lwr1 order by fam) as d + haspostprocessed=true; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceIntersectionNode.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceIntersectionNode.java new file mode 100644 index 0000000..edb3984 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceIntersectionNode.java @@ -0,0 +1,13 @@ +package org.gcube.dataanalysis.executor.nodes.transducers; + +import org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsIntersector; + +public class OccurrenceIntersectionNode extends OccurrenceMergingNode { + + public OccurrenceIntersectionNode() { + super.processor = new OccurrencePointsIntersector(); + } + + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceMergingNode.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceMergingNode.java new file mode 100644 index 0000000..b94a988 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceMergingNode.java @@ -0,0 +1,139 @@ +package org.gcube.dataanalysis.executor.nodes.transducers; + +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsMerger; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.job.management.QueueJobManager; +import org.hibernate.SessionFactory; + +public class OccurrenceMergingNode extends ActorNode { + + // variables + protected AlgorithmConfiguration currentconfig; + protected SessionFactory dbHibConnection; + OccurrencePointsMerger processor; + public int prevbroadcastTimePeriod; + public int prevmaxNumberOfStages; + public int prevmaxMessages; + float status; + + @Override + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON}; + return p; + } + + @Override + public String getName() { + return "OCCURRENCE_PROCESSOR"; + } + + @Override + public String getDescription() { + return processor.getDescription(); + } + + @Override + public List getInputParameters() { + return processor.getInputParameters(); + } + + @Override + public StatisticalType getOutput() { + return processor.getOutput(); + } + + @Override + public void initSingleNode(AlgorithmConfiguration config) { + + } + + @Override + public float getInternalStatus() { + return 0; + } + + public OccurrenceMergingNode() { + processor = new OccurrencePointsMerger(); + } + + @Override + public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStarIndex, int numberOfRightElementsToProcess, boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) { + + try{ + status = 0; + AlgorithmConfiguration config = Transformations.restoreConfig(nodeConfigurationFileObject); + config.setConfigPath(sandboxFolder); + processor.setConfiguration(config); + AnalysisLogger.getLogger().info("Initializing variables"); + processor.init(); + AnalysisLogger.getLogger().info("Initializing DB"); + processor.initDB(false); + status = 0.5f; + processor.takeRange(leftStartIndex, numberOfLeftElementsToProcess, rightStarIndex, numberOfRightElementsToProcess); + processor.computeRange(); + status = 1f; + }catch(Exception e){ + e.printStackTrace(); + System.err.println("warning: error in node execution "+e.getLocalizedMessage()); + return -1; + } + finally{ + stop(); + } + return 0; + } + + @Override + public void setup(AlgorithmConfiguration config) throws Exception { + processor.setConfiguration(config); + processor.init(); + processor.initDB(true); + processor.takeFullRanges(); + prevmaxMessages=D4ScienceDistributedProcessing.maxMessagesAllowedPerJob; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=100; + prevbroadcastTimePeriod = QueueJobManager.broadcastTimePeriod; + QueueJobManager.broadcastTimePeriod=4*3600000; + prevmaxNumberOfStages = QueueJobManager.maxNumberOfStages; + QueueJobManager.maxNumberOfStages=100000; + } + + @Override + public int getNumberOfRightElements() { + return processor.getNumRightObjects(); + } + + @Override + public int getNumberOfLeftElements() { + return processor.getNumLeftObjects(); + } + + @Override + public void stop() { + processor.shutdown(); + } + + @Override + public void postProcess(boolean manageDuplicates, boolean manageFault) { + QueueJobManager.broadcastTimePeriod=prevbroadcastTimePeriod; + QueueJobManager.maxNumberOfStages=prevmaxNumberOfStages; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=prevmaxMessages; + + processor.shutdown(); + try { + processor.postProcess(); + } catch (Exception e) { + AnalysisLogger.getLogger().info("Postprocessing Inapplicable"); + } + + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceSubtractionNode.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceSubtractionNode.java new file mode 100644 index 0000000..751850e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/OccurrenceSubtractionNode.java @@ -0,0 +1,13 @@ +package org.gcube.dataanalysis.executor.nodes.transducers; + +import org.gcube.dataanalysis.ecoengine.transducers.OccurrencePointsSubtraction; + +public class OccurrenceSubtractionNode extends OccurrenceMergingNode { + + public OccurrenceSubtractionNode() { + super.processor = new OccurrencePointsSubtraction(); + } + + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymBiodiv.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymBiodiv.java new file mode 100644 index 0000000..1c89c5a --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymBiodiv.java @@ -0,0 +1,71 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class BionymBiodiv extends BionymFlexibleWorkflowTransducer { + + @Override + public String getName() { + return "BIONYM_BIODIV"; + } + + @Override + public String getDescription() { + return "An algorithm implementing BiOnym oriented to Biodiversity Taxa Names Matching with a predefined and optimized workflow. This version applies in sequence the following Matchers: GSay (thr:0.6, maxRes:10), FuzzyMatcher (thr:0.6, maxRes:10), Levenshtein (thr:0.4, maxRes:10), Trigram (thr:0.4, maxRes:10). BiOnym is a flexible workflow approach to taxon name matching. The workflow allows to activate several taxa names matching algorithms and to get the list of possible transcriptions for a list of input raw species names with possible authorship indication."; + } + + @Override + public List buildMatcherList(AlgorithmConfiguration config, String sandboxFolder, HashMap globalparameters){ + //use the default matchers + return null; + } + + @Override + public List getInputParameters() { + List templateLWRInput = new ArrayList(); + templateLWRInput.add(TableTemplates.GENERIC); + + InputTable p1 = new InputTable(templateLWRInput, originTableParam, "Input table containing raw taxa names that you want to match", "bionym"); + ColumnType p2 = new ColumnType(originTableParam, rawnamesColumnParam, "The column containing the raw taxa names with or without authoship information", "rawnames", false); + ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, destinationTableParam, "name of the table that will contain the matches", "bion_"); + PrimitiveType p4 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, destinationTableLableParam, "Name of the table which will contain the matches", "bionout"); + PrimitiveType p5 = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinDataSources.values(), PrimitiveTypes.ENUMERATED, YasmeenGlobalParameters.taxaAuthorityFileParam, "The reference dataset to use", "" +YasmeenGlobalParameters.BuiltinDataSources.FISHBASE); + PrimitiveType p6 = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinParsers.values(), PrimitiveTypes.ENUMERATED, YasmeenGlobalParameters.parserNameParam, "The Species - Authority parser", "" + YasmeenGlobalParameters.BuiltinParsers.SIMPLE); + PrimitiveType p7 = new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, YasmeenGlobalParameters.activatePreParsingProcessing,"Use preparsing rules to correct common errors","true"); + PrimitiveType p8 = new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, YasmeenGlobalParameters.useStemmedGenusAndSpecies,"Process using Genus and Species names without declension","false"); + PrimitiveType p9 = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, YasmeenGlobalParameters.overallMaxResults,"The maximum number of matching candidates per each raw input species","10"); + + List parameters = new ArrayList(); + parameters.add(p1); + parameters.add(p3); + parameters.add(p2); + parameters.add(p4); + parameters.add(p5); + parameters.add(p6); + parameters.add(p7); + parameters.add(p8); + parameters.add(p9); + + DatabaseType.addDefaultDBPars(parameters); + + return parameters; + + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymFlexibleWorkflowTransducer.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymFlexibleWorkflowTransducer.java new file mode 100644 index 0000000..38a2861 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymFlexibleWorkflowTransducer.java @@ -0,0 +1,438 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.job.management.QueueJobManager; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherOutput; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.SingleEntry; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.FuzzyMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.GsayMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.LevensteinMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.MixedLexicalMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.SoundexMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.TrigramMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.workflows.BiOnymWF; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; +import org.gcube.dataanalysis.executor.scripts.OSCommand; +import org.hibernate.SessionFactory; + +public class BionymFlexibleWorkflowTransducer extends ActorNode { + //SM parameters + protected AlgorithmConfiguration currentconfig; + protected SessionFactory dbconnection; + boolean haspostprocessed = false; + float status; + public int prevbroadcastTimePeriod; + public int prevmaxNumberOfStages; + public int prevmaxMessages; + String destinationTable; + String destinationTableLabel; + + //Table + public static String headers = "SOURCE_DATA,TARGET_DATA_SCIENTIFIC_NAME,TARGET_DATA_AUTHORITY,MATCHING_SCORE,TARGET_DATA_SOURCE,TARGET_DATA_ID"; + private static String createOutputTable = "CREATE TABLE %1$s (SOURCE_DATA character varying(255), TARGET_DATA_SCIENTIFIC_NAME character varying(255), TARGET_DATA_AUTHORITY character varying(255), MATCHING_SCORE real,TARGET_DATA_SOURCE character varying, TARGET_DATA_ID character varying)"; + public int rawnamescount = 0; + + //Algorithm + public static String destinationTableParam = "OutputTable"; + public static String destinationTableLableParam = "OutputTableLabel"; + public static String originTableParam = "RawTaxaNamesTable"; + public static String rawnamesColumnParam = "RawNamesColumn"; + public static String matcherParamPrefix = "Matcher"; + public static String thresholdParamPrefix = "Threshold"; + public static String maxresultsParamPrefix = "MaxResults"; + public static int maxMatchersInterface = 5; + public static int maxMatchersReal = 20; + + public static Matcher getEnum2Matcher(YasmeenGlobalParameters.BuiltinMatchers matcher, String sandboxFolder, double threshold, int maxResults, HashMap parameters){ + + switch(matcher){ + case GSAy: + return new GsayMatcher(sandboxFolder, threshold, maxResults,parameters); + case FUZZYMATCH: + return new FuzzyMatcher(sandboxFolder, threshold, maxResults,parameters); + case LEVENSHTEIN: + return new LevensteinMatcher(sandboxFolder, threshold, maxResults,parameters); + case TRIGRAM: + return new TrigramMatcher(sandboxFolder, threshold, maxResults,parameters); + case SOUNDEX: + return new SoundexMatcher(sandboxFolder, threshold, maxResults,parameters); + case LEV_SDX_TRIG: + return new MixedLexicalMatcher(sandboxFolder, threshold, maxResults,parameters); + case NONE: + return null; + default: + return null; + } + + } + + public List buildMatcherList(AlgorithmConfiguration config, String sandboxFolder, HashMap globalparameters){ + //rebuild the matchers + List matchers = new ArrayList(); + for (int i=1;i<=maxMatchersReal;i++){ + String matchername = config.getParam(matcherParamPrefix+"_"+i); + String threshold = config.getParam(thresholdParamPrefix+"_"+i); + String maxResultS = config.getParam(maxresultsParamPrefix+"_"+i); + if (matchername!=null){ + AnalysisLogger.getLogger().debug("Matcher "+i+" name: "+matchername); + AnalysisLogger.getLogger().debug("Matcher "+i+" threshold: "+threshold); + AnalysisLogger.getLogger().debug("Matcher "+i+" max results: "+maxResultS); + double thr = 0.2; + if (threshold!=null) + thr = Double.parseDouble(threshold); + int maxResults = 10; + if (maxResultS!=null) + maxResults = Integer.parseInt(maxResultS); + + Matcher m = getEnum2Matcher(YasmeenGlobalParameters.BuiltinMatchers.valueOf(matchername),sandboxFolder,thr,maxResults, globalparameters); + if (m!=null) + matchers.add(m); + } + } + return matchers; + } + + @Override + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON }; + return p; + } + + @Override + public String getName() { + return "BIONYM"; + } + + @Override + public String getDescription() { + return "An algorithm implementing BiOnym, a flexible workflow approach to taxon name matching. The workflow allows to activate several taxa names matching algorithms and to get the list of possible transcriptions for a list of input raw species names with possible authorship indication."; + } + + + + @Override + public List getInputParameters() { + List templateLWRInput = new ArrayList(); + templateLWRInput.add(TableTemplates.GENERIC); + + InputTable p1 = new InputTable(templateLWRInput, originTableParam, "Input table containing raw taxa names that you want to match", "byonym"); + ColumnType p2 = new ColumnType(originTableParam, rawnamesColumnParam, "The column containing the raw taxa names with or without authoship information", "rawnames", false); + ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, destinationTableParam, "name of the table that will contain the matches", "bion_"); + PrimitiveType p4 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, destinationTableLableParam, "Name of the table which will contain the matches", "bionout"); + + PrimitiveType p5 = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinDataSources.values(), PrimitiveTypes.ENUMERATED, YasmeenGlobalParameters.taxaAuthorityFileParam, "The reference dataset to use", "" +YasmeenGlobalParameters.BuiltinDataSources.FISHBASE); + PrimitiveType p6 = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinParsers.values(), PrimitiveTypes.ENUMERATED, YasmeenGlobalParameters.parserNameParam, "The Species - Authority parser", "" + YasmeenGlobalParameters.BuiltinParsers.SIMPLE); + PrimitiveType p7 = new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, YasmeenGlobalParameters.activatePreParsingProcessing,"Use preparsing rules to correct common errors","true"); + PrimitiveType p8 = new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, YasmeenGlobalParameters.useStemmedGenusAndSpecies,"Process using Genus and Species names without declension","false"); + PrimitiveType p9 = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.Performance.values(), PrimitiveTypes.ENUMERATED, YasmeenGlobalParameters.performanceParam, "A trade-off between recognition speed and accuracy. Max speed corresponds to search for strings with the same length only.", "" + YasmeenGlobalParameters.Performance.MAX_ACCURACY); + + List parameters = new ArrayList(); + parameters.add(p1); + parameters.add(p3); + parameters.add(p2); + parameters.add(p4); + parameters.add(p5); + parameters.add(p6); + parameters.add(p7); + parameters.add(p8); + parameters.add(p9); + + PrimitiveType pgsay = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinMatchers.values(), PrimitiveTypes.ENUMERATED, matcherParamPrefix+"_"+1, "Choose a Matcher", YasmeenGlobalParameters.BuiltinMatchers.GSAy.name()); + PrimitiveType pthrgsay = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, thresholdParamPrefix+"_"+1 ,"Threshold","0.6",true); + PrimitiveType pmrgsay = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, maxresultsParamPrefix+"_"+1,"The maximum number of matching candidates per each raw input species","10"); + + PrimitiveType ptaxa = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinMatchers.values(), PrimitiveTypes.ENUMERATED, matcherParamPrefix+"_"+2, "Choose a Matcher", YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name()); + PrimitiveType pthrtaxa = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, thresholdParamPrefix+"_"+2 ,"Threshold","0.6",true); + PrimitiveType pmrtaxa = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, maxresultsParamPrefix+"_"+2,"The maximum number of matching candidates per each raw input species","10"); + + PrimitiveType plev = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinMatchers.values(), PrimitiveTypes.ENUMERATED, matcherParamPrefix+"_"+3, "Choose a Matcher", YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + PrimitiveType pthrlev = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, thresholdParamPrefix+"_"+3 ,"Threshold","0.4",true); + PrimitiveType pmrlev = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, maxresultsParamPrefix+"_"+3,"The maximum number of matching candidates per each raw input species","5"); + + PrimitiveType ptrig = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinMatchers.values(), PrimitiveTypes.ENUMERATED, matcherParamPrefix+"_"+4, "Choose a Matcher", YasmeenGlobalParameters.BuiltinMatchers.TRIGRAM.name()); + PrimitiveType pthrtrig = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, thresholdParamPrefix+"_"+4 ,"Threshold","0.4"); + PrimitiveType pmrtrig = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, maxresultsParamPrefix+"_"+4,"The maximum number of matching candidates per each raw input species","5"); + + parameters.add(pgsay); + parameters.add(pthrgsay); + parameters.add(pmrgsay); + + parameters.add(ptaxa); + parameters.add(pthrtaxa); + parameters.add(pmrtaxa); + + parameters.add(plev); + parameters.add(pthrlev); + parameters.add(pmrlev); + + parameters.add(ptrig); + parameters.add(pthrtrig); + parameters.add(pmrtrig); + + for (int i=5;i<=maxMatchersInterface;i++){ + PrimitiveType p = new PrimitiveType(Enum.class.getName(), YasmeenGlobalParameters.BuiltinMatchers.values(), PrimitiveTypes.ENUMERATED, matcherParamPrefix+"_"+i, "Choose a Matcher (Optional)", YasmeenGlobalParameters.BuiltinMatchers.NONE.name(),true); + PrimitiveType pn = new PrimitiveType(Double.class.getName(), null, PrimitiveTypes.NUMBER, thresholdParamPrefix+"_"+i ,"Threshold (def. 0.2)","0.2",true); + PrimitiveType pr = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, maxresultsParamPrefix+"_"+i,"The maximum number of matching candidates per each raw input species","0"); + parameters.add(p); + parameters.add(pn); + parameters.add(pr); + } + + DatabaseType.addDefaultDBPars(parameters); + + return parameters; + + } + + @Override + public StatisticalType getOutput() { + List template = new ArrayList(); + template.add(TableTemplates.GENERIC); + OutputTable p = new OutputTable(template, destinationTableLabel, destinationTable, "Output table"); + return p; + } + + @Override + public void initSingleNode(AlgorithmConfiguration config) { + } + + @Override + public float getInternalStatus() { + return status; + } + + @Override + public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStartIndex, int numberOfRightElementsToProcess,boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) { + String uuid = (""+UUID.randomUUID()).replace("-", ""); + String inputParserFile = "inputParser"+uuid+".txt"; + String outputParserFile = "outputParser"+uuid+".txt"; + try { + status = 0; + long t0 = System.currentTimeMillis(); + // rebuild variables + System.out.println("Restoring configuration"); + AlgorithmConfiguration config = Transformations.restoreConfig(new File(sandboxFolder, nodeConfigurationFileObject).getAbsolutePath()); + config.setConfigPath(sandboxFolder); + dbconnection = DatabaseUtils.initDBSession(config); + + String destinationTable = config.getParam(destinationTableParam); + String originTable = config.getParam(originTableParam); + String rawnamesColumn = config.getParam(rawnamesColumnParam); + + String parser = config.getParam(YasmeenGlobalParameters.parserNameParam); + String accuracyvsspeed = config.getParam(YasmeenGlobalParameters.performanceParam); + String reference = config.getParam(YasmeenGlobalParameters.taxaAuthorityFileParam); + String doPreprocess =config.getParam(YasmeenGlobalParameters.activatePreParsingProcessing); + String usestemming= config.getParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies); + String overallMaxResults= config.getParam(YasmeenGlobalParameters.overallMaxResults); + + + System.out.println("Destination Table: " + destinationTable); + System.out.println("Origin Table: " + originTable); + System.out.println("Column of names: " + rawnamesColumn); + System.out.println("Parser to use: " + parser); + System.out.println("Accuracy vs Speed: " + accuracyvsspeed); + System.out.println("Reference Dataset: " + reference); + System.out.println("Do Preprocessing: " + doPreprocess); + System.out.println("Use Stemming:" + usestemming); + System.out.println("Overall MaxResults:" + overallMaxResults); + + //prepare the WF + HashMap globalparameters = new HashMap(); + + globalparameters.put(YasmeenGlobalParameters.parserInputFileParam, inputParserFile); + globalparameters.put(YasmeenGlobalParameters.parserOutputFileParam, outputParserFile); + globalparameters.put(YasmeenGlobalParameters.activatePreParsingProcessing, doPreprocess); + globalparameters.put(YasmeenGlobalParameters.parserNameParam, parser); + globalparameters.put(YasmeenGlobalParameters.performanceParam, accuracyvsspeed); + globalparameters.put(YasmeenGlobalParameters.taxaAuthorityFileParam, reference); + globalparameters.put(YasmeenGlobalParameters.useStemmedGenusAndSpecies, usestemming); + System.out.println("Configuration Restored! - Time: "+(System.currentTimeMillis()-t0)); + + // retrieve the list of names to process + long t1 = System.currentTimeMillis(); + System.out.println("Retrieving names to process"); + String query = DatabaseUtils.getDinstictElements(originTable, rawnamesColumn, "")+" offset "+rightStartIndex+" limit "+numberOfRightElementsToProcess; + List rawnames = DatabaseFactory.executeSQLQuery(query, dbconnection); + System.out.println("Retrieved a total of "+rawnames.size()+" species"); + //modification due to the limit and offset on the query + rightStartIndex = 0; + + int end = rightStartIndex + numberOfRightElementsToProcess; + + System.out.println("Processing from "+rightStartIndex+" to "+end); + List rawnamesFiltered = new ArrayList(); + for (int i = rightStartIndex; i < end; i++) { + String raw = "" + rawnames.get(i); +// System.out.println("Taking species:"+raw); + rawnamesFiltered.add(raw.replaceAll("^'", "").replaceAll("'$", "")); + } + int rawscounter = rawnamesFiltered.size(); + System.out.println("Retrieve from DB - Time: "+(System.currentTimeMillis()-t1)); + + long t2 = System.currentTimeMillis(); + System.out.println("Processing " + rawscounter + " species.."); + // prepare the environment + try { + OSCommand.ExecuteGetLine("chmod +x *", null); + } catch (Exception e) { + System.out.println("WARNING: could not change the permissions"); + } + + int overallMR = 10; + if (overallMaxResults!=null) + overallMR=Integer.parseInt(overallMaxResults); + + BiOnymWF bionym = new BiOnymWF(sandboxFolder, overallMR, globalparameters); + + //rebuild the matchers + List matchers = buildMatcherList(config, sandboxFolder, globalparameters); + if (matchers!=null) + bionym.resetMatchers(matchers); + System.out.println("WF Initialization - Time: "+(System.currentTimeMillis()-t2)); + + MatcherOutput output = bionym.executeChainedWorkflow(rawnamesFiltered); + System.out.println("Workflow Executed"); + long t3 = System.currentTimeMillis(); + int nEntries = output.getEntriesNumber(); + List toWrite = new ArrayList(); + for (int i=0;i rawnames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(originTable, rawnamesColumn, ""), dbconnection); + + rawnamescount = rawnames.size(); +// rawnamescount =1; + + AnalysisLogger.getLogger().info("Creating Destination Table " + destinationTable); + try { + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(destinationTable), dbconnection); + } catch (Exception e) { + AnalysisLogger.getLogger().info("Table " + destinationTable + " did not exist"); + } + + DatabaseFactory.executeSQLUpdate(String.format(createOutputTable, destinationTable), dbconnection); + + prevmaxMessages=D4ScienceDistributedProcessing.maxMessagesAllowedPerJob; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=50; + prevbroadcastTimePeriod = QueueJobManager.broadcastTimePeriod; + QueueJobManager.broadcastTimePeriod=30*60000; + prevmaxNumberOfStages = QueueJobManager.maxNumberOfStages; + QueueJobManager.maxNumberOfStages=10000; + + AnalysisLogger.getLogger().info("Destination Table Created! Addressing " + rawnamescount + " names"); + } + + @Override + public int getNumberOfRightElements() { + return rawnamescount; + } + + @Override + public int getNumberOfLeftElements() { + return 1; + } + + @Override + public void stop() { + // if has not postprocessed, then abort the computations by removing the database table + if (!haspostprocessed) { + try { + AnalysisLogger.getLogger().info("The procedure did NOT correctly postprocessed ....Removing Table " + destinationTable + " because of computation stop!"); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(destinationTable), dbconnection); + } catch (Exception e) { + AnalysisLogger.getLogger().info("Table " + destinationTable + " did not exist"); + } + } else + AnalysisLogger.getLogger().info("The procedure has correctly postprocessed: shutting down the connection!"); + if (dbconnection != null) + try { + dbconnection.close(); + } catch (Exception e) { + } + } + + @Override + public void postProcess(boolean manageDuplicates, boolean manageFault) { + QueueJobManager.broadcastTimePeriod=prevbroadcastTimePeriod; + QueueJobManager.maxNumberOfStages=prevmaxNumberOfStages; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=prevmaxMessages; + haspostprocessed = true; + } + + + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymLocalTransducer.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymLocalTransducer.java new file mode 100644 index 0000000..c6d0b19 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymLocalTransducer.java @@ -0,0 +1,201 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.configuration.INFRASTRUCTURE; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; +import org.gcube.dataanalysis.ecoengine.utils.ResourceFactory; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherOutput; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.SingleEntry; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.workflows.BiOnymWF; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; +import org.gcube.dataanalysis.executor.scripts.OSCommand; + +import scala.actors.threadpool.Arrays; + +public class BionymLocalTransducer implements Transducerer { + // BionymFlexibleWorkflowTransducer bionymwf = new BionymFlexibleWorkflowTransducer(); + public float status = 0f; + AlgorithmConfiguration config = null; + public static String speciesName = "SpeciesAuthorName"; + public LinkedHashMap outputmap = new LinkedHashMap(); + + @Override + public List getInputParameters() { + List types = new BionymFlexibleWorkflowTransducer().getInputParameters(); + PrimitiveType p1 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, speciesName, "The scientific name of the species, possibly with authorship", "Gadus morhua (Linnaeus, 1758)"); + types.remove(0); + types.remove(0); + types.remove(0); + types.remove(0); + + types.add(0, p1); + + return types; + } + + @Override + public String getResourceLoad() { + ResourceFactory resourceManager = new ResourceFactory(); + return resourceManager.getResourceLoad(1); + } + + public String getResources() { + if ((status > 0) && (status < 100)) + return ResourceFactory.getResources(100f); + else + return ResourceFactory.getResources(0f); + } + + @Override + public float getStatus() { + return status; + } + + @Override + public INFRASTRUCTURE getInfrastructure() { + return INFRASTRUCTURE.LOCAL; + } + + @Override + public StatisticalType getOutput() { + PrimitiveType p = new PrimitiveType(Map.class.getName(), PrimitiveType.stringMap2StatisticalMap(outputmap), PrimitiveTypes.MAP, "RetrievedTaxaNames", "Retrieved Taxa Names"); + return p; + } + + @Override + public void init() throws Exception { + } + + @Override + public void setConfiguration(AlgorithmConfiguration config) { + this.config = config; + } + + @Override + public void shutdown() { + } + + @Override + public String getDescription() { + return "A fast version of the algorithm implementing BiOnym, a flexible workflow approach to taxon name matching. The workflow allows to activate several taxa names matching algorithms and to get the list of possible transcriptions for a list of input raw species names with possible authorship indication."; + } + + public void compute() throws Exception { + status = 10f; + String uuid = ("" + UUID.randomUUID()).replace("-", ""); + String inputParserFile = "inputParser" + uuid + ".txt"; + String outputParserFile = "outputParser" + uuid + ".txt"; + String sandboxFolder = ""; + try { + String parser = config.getParam(YasmeenGlobalParameters.parserNameParam); + String accuracyvsspeed = config.getParam(YasmeenGlobalParameters.performanceParam); + String reference = config.getParam(YasmeenGlobalParameters.taxaAuthorityFileParam); + String doPreprocess = config.getParam(YasmeenGlobalParameters.activatePreParsingProcessing); + String usestemming = config.getParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies); + String overallMaxResults = config.getParam(YasmeenGlobalParameters.overallMaxResults); + String inputSpecies = config.getParam(speciesName); + sandboxFolder = new File(config.getPersistencePath(), D4ScienceDistributedProcessing.defaultContainerFolder).getAbsolutePath(); + + AnalysisLogger.getLogger().debug("BiOnymLocal-> Species Name: " + inputSpecies); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Parser to use: " + parser); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Accuracy vs Speed: " + accuracyvsspeed); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Reference Dataset: " + reference); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Do Preprocessing: " + doPreprocess); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Use Stemming:" + usestemming); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Overall MaxResults:" + overallMaxResults); + + // prepare the WF + HashMap globalparameters = new HashMap(); + + globalparameters.put(YasmeenGlobalParameters.parserInputFileParam, inputParserFile); + globalparameters.put(YasmeenGlobalParameters.parserOutputFileParam, outputParserFile); + globalparameters.put(YasmeenGlobalParameters.activatePreParsingProcessing, doPreprocess); + globalparameters.put(YasmeenGlobalParameters.parserNameParam, parser); + globalparameters.put(YasmeenGlobalParameters.performanceParam, accuracyvsspeed); + globalparameters.put(YasmeenGlobalParameters.taxaAuthorityFileParam, reference); + globalparameters.put(YasmeenGlobalParameters.useStemmedGenusAndSpecies, usestemming); + globalparameters.put(YasmeenGlobalParameters.staticFilesFolderParam, config.getConfigPath()); + // retrieve the list of names to process + inputSpecies = inputSpecies.replaceAll("^'", "").replaceAll("'$", ""); + List rawnamesFiltered = new ArrayList(); + rawnamesFiltered.add(inputSpecies); + + // prepare the environment + try { + OSCommand.ExecuteGetLine("chmod +x *", null); + } catch (Exception e) { + AnalysisLogger.getLogger().debug("BiOnymLocal-> WARNING: could not change the permissions"); + } + + int overallMR = 10; + if (overallMaxResults != null) + overallMR = Integer.parseInt(overallMaxResults); + + AnalysisLogger.getLogger().debug("BiOnymLocal-> Executing WF"); + BiOnymWF bionym = new BiOnymWF(sandboxFolder, overallMR, globalparameters); + // rebuild the matchers + List matchers = new BionymFlexibleWorkflowTransducer().buildMatcherList(config, sandboxFolder, globalparameters); + if (matchers != null) + bionym.resetMatchers(matchers); + + AnalysisLogger.getLogger().debug("BiOnymLocal-> WorkFlow: "); + int mcounter = 1; + for (Matcher matcher:matchers){ + AnalysisLogger.getLogger().debug("BiOnymLocal-> "+mcounter+": "+matcher); + mcounter++; + } + status = 20f; + MatcherOutput output = bionym.executeChainedWorkflow(rawnamesFiltered); + AnalysisLogger.getLogger().debug("BiOnymLocal-> Workflow Executed"); + + status = 70f; + int nEntries = output.getEntriesNumber(); + outputmap.put("HEADER", BionymFlexibleWorkflowTransducer.headers.toString()); + + for (int i = 0; i < nEntries; i++) { + SingleEntry se = output.getEntry(i); + // "SOURCE_DATA,TARGET_DATA_SCIENTIFIC_NAME,TARGET_DATA_AUTHORITY,MATCHING_SCORE,TARGET_DATA_SOURCE,TARGET_DATA_ID"; + String[] srow = new String[6]; + srow[0] = "\""+se.originalName+"\""; + srow[1] = "\""+se.targetScientificName+"\""; + srow[2] = "\""+se.targetAuthor+"\""; + srow[3] = "\"" + se.matchingScore+"\""; + srow[4] = "\""+reference+"\""; + srow[5] = "\""+se.targetID+"\""; + + outputmap.put("" + (i + 1), Arrays.toString(srow)); + } + if (nEntries == 0) + AnalysisLogger.getLogger().debug("BiOnymLocal-> Warning no output found!"); + + AnalysisLogger.getLogger().debug("BiOnymLocal-> map of outputs:\n" + outputmap); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + + new File(sandboxFolder,inputParserFile).delete(); + new File(sandboxFolder,outputParserFile).delete(); + + AnalysisLogger.getLogger().debug("BiOnymLocal-> deleting auxiliary files"); + AnalysisLogger.getLogger().debug("BiOnymLocal-> shutting down"); + status = 100f; + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymWorkflow.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymWorkflow.java new file mode 100644 index 0000000..2cdd83b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/BionymWorkflow.java @@ -0,0 +1,436 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.ALG_PROPS; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.ColumnType; +import org.gcube.dataanalysis.ecoengine.datatypes.DatabaseType; +import org.gcube.dataanalysis.ecoengine.datatypes.InputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.OutputTable; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.ServiceType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.PrimitiveTypes; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.ServiceParameters; +import org.gcube.dataanalysis.ecoengine.datatypes.enumtypes.TableTemplates; +import org.gcube.dataanalysis.ecoengine.interfaces.ActorNode; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseFactory; +import org.gcube.dataanalysis.ecoengine.utils.DatabaseUtils; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.ecoengine.utils.Tuple; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.job.management.QueueJobManager; +import org.gcube.dataanalysis.executor.scripts.OSCommand; +import org.hibernate.SessionFactory; + +import com.thoughtworks.xstream.XStream; + +public class BionymWorkflow extends ActorNode { + protected AlgorithmConfiguration currentconfig; + protected SessionFactory dbconnection; + private static String createOutputTable = "CREATE TABLE %1$s (inputname character varying(255), suggestion character varying(255), score real)"; + int rawnamescount = 0; + public int prevbroadcastTimePeriod; + public int prevmaxNumberOfStages; + public int prevmaxMessages; + + String destinationTable; + public static String destinationTableParam = "OutputTable"; + public static String destinationTableLable = "OutputTableLabel"; + String originTable; + public static String originTableParam = "RawTaxaNamesTable"; + String rawnamesColumn; + public static String rawnamesColumnParam = "RawNamesColumn"; + String parser; + public static String parserParam = "Parser"; + String reference; + public static String referenceParam = "ReferenceDataset"; + String soundexweight; + public static String soundexweightParam = "SoundexVSEditDist"; + String preprocessor; + public static String doPreprocessParam = "Preprocess"; + float status; + public static String maxMatchesParam= "MaxMatches"; + int maxMatches=10; + public static String pruningThresholdParam= "PruningThreshold"; + float pruningThreshold=0.4f; + + static String headers = "inputname,suggestion,score"; + boolean haspostprocessed = false; + + @Override + public ALG_PROPS[] getProperties() { + ALG_PROPS[] p = { ALG_PROPS.PHENOMENON_VS_PARALLEL_PHENOMENON }; + return p; + } + + @Override + public String getName() { + return "BIONYM"; + } + + @Override + public String getDescription() { + return "An algorithm implementing BiOnym, a flexible workflow approach to taxon name matching. The workflow allows to activate several taxa names matching algorithms and to get the list of possible transcriptions for a list of input raw species names with possible authorship indication."; + } + + @Override + public List getInputParameters() { + List templateLWRInput = new ArrayList(); + templateLWRInput.add(TableTemplates.GENERIC); + + InputTable p1 = new InputTable(templateLWRInput, originTableParam, "Input table containing raw taxa names that you want to match", "byonym"); + ColumnType p2 = new ColumnType(originTableParam, rawnamesColumnParam, "The column containing the raw taxa names with or without authoship information", "rawnames", false); + ServiceType p3 = new ServiceType(ServiceParameters.RANDOMSTRING, destinationTableParam, "name of the table that will contain the matches", "bion_"); + PrimitiveType p4 = new PrimitiveType(String.class.getName(), null, PrimitiveTypes.STRING, destinationTableLable, "Name of the table which will contain the matches", "bionout"); + PrimitiveType p5 = new PrimitiveType(Enum.class.getName(), CometMatcherManager.Reference.values(), PrimitiveTypes.ENUMERATED, referenceParam, "The reference dataset to use", "" + CometMatcherManager.Reference.FISHBASE); + PrimitiveType p6 = new PrimitiveType(Enum.class.getName(), EVBPreprocessing.Preprocessors.values(), PrimitiveTypes.ENUMERATED, doPreprocessParam, "Set a preprocessing approach for the raw strings", "" + EVBPreprocessing.Preprocessors.EXPERT_RULES); + PrimitiveType p7 = new PrimitiveType(Enum.class.getName(), CometMatcherManager.Weights.values(), PrimitiveTypes.ENUMERATED, soundexweightParam, "Set the use of soundex vs edit distance approaches to string matching", "" + CometMatcherManager.Weights.EDIT_DISTANCE); + PrimitiveType p8 = new PrimitiveType(Enum.class.getName(), CometMatcherManager.Parsers.values(), PrimitiveTypes.ENUMERATED, parserParam, "Set the genus-species-author parser to use", "" + CometMatcherManager.Parsers.SIMPLE); + PrimitiveType p9 = new PrimitiveType(Float.class.getName(), null, PrimitiveTypes.NUMBER, pruningThresholdParam,"Pruning threshold for the output scores (from 0 to 1)","0.4"); + PrimitiveType p10 = new PrimitiveType(Integer.class.getName(), null, PrimitiveTypes.NUMBER, maxMatchesParam,"Maximum number of matches to report per raw string","10"); + + + List parameters = new ArrayList(); + parameters.add(p1); + parameters.add(p3); + parameters.add(p2); + parameters.add(p4); + parameters.add(p5); + parameters.add(p6); + parameters.add(p7); + parameters.add(p8); + parameters.add(p9); + parameters.add(p10); + + DatabaseType.addDefaultDBPars(parameters); + + return parameters; + + } + + @Override + public StatisticalType getOutput() { + List template = new ArrayList(); + template.add(TableTemplates.GENERIC); + OutputTable p = new OutputTable(template, destinationTableLable, destinationTable, "Output table"); + return p; + } + + @Override + public void initSingleNode(AlgorithmConfiguration config) { + + + } + + @Override + public float getInternalStatus() { + + return status; + } + + List> matchedTuples = new ArrayList>(); + List> unmatchedTuples = new ArrayList>(); + + public void filterMatchedTuples(List> tuples){ + unmatchedTuples = null; + unmatchedTuples = new ArrayList>(); + + for (Tuple tuple:tuples){ + double score = 0; + if (tuple.getElements().size()>2){ + try{score = Double.parseDouble(tuple.getElements().get(2));}catch(Exception e){ + e.printStackTrace(); + } + } + if (score==1) + matchedTuples.add(tuple); + else + unmatchedTuples.add(tuple); + } + + } + + public StringBuffer executeBionymWorkflow(String rawspeciesname, String sandboxfolder, String preprocessor, boolean[] enablematchers, float soundexweightF, int maxResults, float pruningThreshold) throws Exception { + StringBuffer sb = new StringBuffer(); + List rawnames = new ArrayList(); + rawnames.add(rawspeciesname); + + List> preprocessednames = new ArrayList>(); + + // preprocessing + switch (EVBPreprocessing.Preprocessors.valueOf(preprocessor)) { + case EXPERT_RULES: + EVBPreprocessing preprocess = new EVBPreprocessing(); + preprocessednames = preprocess.preprocess(parser, sandboxfolder, rawnames); + break; + default: + preprocessednames = EVBPreprocessing.populateTuples(rawnames); + break; + } + + filterMatchedTuples(preprocessednames); + + for (int i = 0; i < enablematchers.length; i++) { + // standard WF + if (i == 0) { + CometMatcherManager comet = new CometMatcherManager(); + List> cometoutput = comet.match(parser, reference, sandboxfolder, unmatchedTuples, soundexweightF,maxResults); + filterMatchedTuples(cometoutput); + } + } + + //add all non exact matches + matchedTuples.addAll(unmatchedTuples); + int msize = matchedTuples.size(); + + if (msize>0) { + + for (int i = 0; i < msize; i++) { + Tuple t = matchedTuples.get(i); + String scoreS = t.getElements().get(2); + Float score = (scoreS !=null)? Float.parseFloat(scoreS):0; + + if (score>=pruningThreshold){ + String spname = t.getElements().get(0); + String authorname = t.getElements().get(1); + if (authorname.length()>0) + spname +=" ("+authorname+")"; + + sb.append("('" + rawspeciesname + "','" + spname + "','" + scoreS + "')"); + if (i < msize - 1) + sb.append(","); + } + } + } + String sbstring = sb.toString().trim(); + int ssize = sbstring.length(); + + if (sbstring.endsWith(",")){ + System.out.println("Deleting final comma.."); + sb= new StringBuffer(sbstring.substring(0,ssize-1)); + } + return sb; + } + + @Override + public int executeNode(int leftStartIndex, int numberOfLeftElementsToProcess, int rightStartIndex, int numberOfRightElementsToProcess,boolean duplicate, String sandboxFolder, String nodeConfigurationFileObject, String logfileNameToProduce) { + try { + status = 0; + long t0 = System.currentTimeMillis(); + // rebuild variables + System.out.println("Restoring configuration"); + AlgorithmConfiguration config = Transformations.restoreConfig(new File(sandboxFolder, nodeConfigurationFileObject).getAbsolutePath()); + config.setConfigPath(sandboxFolder); + dbconnection = DatabaseUtils.initDBSession(config); + destinationTable = config.getParam(destinationTableParam); + originTable = config.getParam(originTableParam); + rawnamesColumn = config.getParam(rawnamesColumnParam); + parser = config.getParam(parserParam); + reference = config.getParam(referenceParam); + soundexweight = config.getParam(soundexweightParam); + preprocessor = config.getParam(doPreprocessParam); + String maxMatchesS = config.getParam(maxMatchesParam); + maxMatches= (maxMatchesS==null)?10:Integer.parseInt(maxMatchesS); + String pruningThrS = config.getParam(pruningThresholdParam); + pruningThreshold = (pruningThrS==null)?0.4f:Float.parseFloat(pruningThrS); + + System.out.println("Destination Table: " + destinationTable); + System.out.println("Origin Table: " + originTable); + System.out.println("Column of names: " + rawnamesColumn); + System.out.println("Parser to use: " + parser); + System.out.println("Reference Dataset: " + reference); + System.out.println("Soundex Preference: " + soundexweight); + System.out.println("Preprocessor:" + preprocessor); + System.out.println("Pruning threshold:" + pruningThreshold); + System.out.println("Number of Matches:" + maxMatches); + + float soundexweightF = 0.5f; + switch (CometMatcherManager.Weights.valueOf(soundexweight)) { + case SOUNDEX: + soundexweightF = 1f; + break; + case EDIT_DISTANCE: + soundexweightF = 0f; + break; + case MIXED: + soundexweightF = 0.5f; + break; + default: + soundexweightF = 0.5f; + break; + } + + // retrieve the list of names to process + System.out.println("Retrieving names to process"); + List rawnames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(originTable, rawnamesColumn, ""), dbconnection); + System.out.println("Retrieved a total of "+rawnames.size()+" species"); + + int end = rightStartIndex + numberOfRightElementsToProcess; + + System.out.println("Processing from "+rightStartIndex+" to "+end); + List rawnamesFiltered = new ArrayList(); + for (int i = rightStartIndex; i < end; i++) { + rawnamesFiltered.add("" + rawnames.get(i)); + } + // prepare the environment + try { +// OSCommand.ExecuteGetLine("chmod +x * | whoami", null); + OSCommand.ExecuteGetLine("chmod +x *", null); + } catch (Exception e) { + System.out.println("WARNING: could not change the permissions"); + } + int rawscounter = 0; + System.out.println("Processing raw names"); + for (String rawname : rawnamesFiltered) { + rawname = rawname.replace("'", "").replace("\"", ""); + System.out.println("Processing species: "+rawname); + StringBuffer sb = executeBionymWorkflow(rawname, sandboxFolder, preprocessor, new boolean[] { true }, soundexweightF,maxMatches,pruningThreshold); + System.out.println("Processed species: "+rawname); + if (sb.length() > 0) { + rawscounter++; + System.out.println("Inserting results onto the table "+destinationTable); + String insertQuery = DatabaseUtils.insertFromBuffer(destinationTable, headers, sb); + System.out.println("Insert Query: " + insertQuery); + System.out.println("Inserting values for " + rawname); + DatabaseFactory.executeSQLUpdate(insertQuery, dbconnection); + System.out.println("Successfully Inserted values for " + rawname); + } +// Thread.sleep(10000); + } + + System.out.println("The procedure finished successfully. Processed " + rawscounter + " species."); + System.out.println("Elapsed Time " + (System.currentTimeMillis() - t0) + " ms"); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("warning: error in node execution " + e.getLocalizedMessage()); + System.err.println("Error in node execution " + e.getLocalizedMessage()); + return -1; + } finally { + if (dbconnection != null) + try { + dbconnection.close(); + } catch (Exception e) { + } + status = 1f; + } + + return 0; + } + + @Override + public void setup(AlgorithmConfiguration config) throws Exception { + haspostprocessed = false; + AnalysisLogger.getLogger().info("Initializing DB Connection"); + dbconnection = DatabaseUtils.initDBSession(config); + destinationTable = config.getParam(destinationTableParam); + originTable = config.getParam(originTableParam); + rawnamesColumn = config.getParam(rawnamesColumnParam); + + List rawnames = DatabaseFactory.executeSQLQuery(DatabaseUtils.getDinstictElements(originTable, rawnamesColumn, ""), dbconnection); + rawnamescount = rawnames.size(); +// rawnamescount =1; + AnalysisLogger.getLogger().info("Creating Destination Table " + destinationTable); + try { + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(destinationTable), dbconnection); + } catch (Exception e) { + AnalysisLogger.getLogger().info("Table " + destinationTable + " did not exist"); + } + + DatabaseFactory.executeSQLUpdate(String.format(createOutputTable, destinationTable), dbconnection); + + prevmaxMessages=D4ScienceDistributedProcessing.maxMessagesAllowedPerJob; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=1; + prevbroadcastTimePeriod = QueueJobManager.broadcastTimePeriod; + QueueJobManager.broadcastTimePeriod=4*3600000; + prevmaxNumberOfStages = QueueJobManager.maxNumberOfStages; + QueueJobManager.maxNumberOfStages=10000; + + AnalysisLogger.getLogger().info("Destination Table Created! Addressing " + rawnamescount + " names"); + + } + + @Override + public int getNumberOfRightElements() { + return rawnamescount; + } + + @Override + public int getNumberOfLeftElements() { + return 1; + } + + @Override + public void stop() { + // if has not postprocessed, then abort the computations by removing the database table + if (!haspostprocessed) { + try { + AnalysisLogger.getLogger().info("The procedure did NOT correctly postprocessed ....Removing Table " + destinationTable + " because of computation stop!"); + DatabaseFactory.executeSQLUpdate(DatabaseUtils.dropTableStatement(destinationTable), dbconnection); + } catch (Exception e) { + AnalysisLogger.getLogger().info("Table " + destinationTable + " did not exist"); + } + } else + AnalysisLogger.getLogger().info("The procedure has correctly postprocessed: shutting down the connection!"); + if (dbconnection != null) + try { + dbconnection.close(); + } catch (Exception e) { + } + } + + @Override + public void postProcess(boolean manageDuplicates, boolean manageFault) { + QueueJobManager.broadcastTimePeriod=prevbroadcastTimePeriod; + QueueJobManager.maxNumberOfStages=prevmaxNumberOfStages; + D4ScienceDistributedProcessing.maxMessagesAllowedPerJob=prevmaxMessages; + haspostprocessed = true; + } + + + public static void mainTEST(String[] args) throws Exception { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + String sandbox = "./PARALLEL_PROCESSING"; + String configfile = "testconfig.cfg"; + config.setPersistencePath(sandbox); + /* + config.setParam("DatabaseUserName", "gcube"); + config.setParam("DatabasePassword", "d4science2"); + config.setParam("DatabaseURL", "jdbc:postgresql://146.48.87.169/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); +*/ + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(BionymWorkflow.destinationTableParam, "taxamatchoutputlocal"); + config.setParam(BionymWorkflow.destinationTableLable, "taxamatchoutputlabel"); + config.setParam(BionymWorkflow.originTableParam, "taxamatchinput"); + config.setParam(BionymWorkflow.rawnamesColumnParam, "rawstrings"); + config.setParam(BionymWorkflow.parserParam, CometMatcherManager.Parsers.SIMPLE.name()); + config.setParam(BionymWorkflow.referenceParam, CometMatcherManager.Reference.ASFIS.name()); + config.setParam(BionymWorkflow.soundexweightParam, CometMatcherManager.Weights.EDIT_DISTANCE.name()); + config.setParam(BionymWorkflow.doPreprocessParam , EVBPreprocessing.Preprocessors.EXPERT_RULES.name()); + config.setParam(BionymWorkflow.maxMatchesParam , "10"); + + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + BufferedWriter oos = new BufferedWriter(new FileWriter(new File(sandbox, configfile))); + oos.write(new XStream().toXML(config)); + oos.close(); + + new BionymWorkflow().setup(config); + +// new BionymWorkflow().executeNode(0, 1, 0, 915, false, sandbox, configfile, "test.log"); + new BionymWorkflow().executeNode(0, 1, 0, 1, false, sandbox, configfile, "test.log"); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/CometMatcherManager.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/CometMatcherManager.java new file mode 100644 index 0000000..15a546c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/CometMatcherManager.java @@ -0,0 +1,200 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.ecoengine.utils.Tuple; + +public class CometMatcherManager { + + public enum Parsers { + SIMPLE, GNI + } + + public enum Reference { + ASFIS, FISHBASE, OBIS + } + + public enum Weights { + SOUNDEX, EDIT_DISTANCE, MIXED + } + + public static void cometParse(String pathToComet, String parser, String outFile, String inFile) throws Exception { + if (!pathToComet.endsWith("/")) + pathToComet += "/"; + String execution = "java -Xmx512m -Xmx1024m -jar " + pathToComet + "SpeciMEn1.0.71.jar -pt 6 -parser " + parser + " -parseOnly -parserOutFile " + outFile + " -inFile " + inFile; + System.out.println("Executing: " + execution); + Process process = null; + + try{ + process = Runtime.getRuntime().exec(execution); + + + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = br.readLine(); + System.out.println(line); + while (line != null) { + line = br.readLine(); + System.out.println(line); + } + }catch(Exception e){ + System.out.println("Unable to execute the program"); + throw e; + } + finally{ + if (process!=null) + process.destroy(); + } + } + + public static void cometMatch(String pathToComet, String parser, String reference, String outFile, String inFile, float sxnw,int maxresults) throws Exception { + if (!pathToComet.endsWith("/")) + pathToComet += "/"; + // java -Xmx512m -Xmx1024m -jar SpeciMEn1.0.71.jar -pt 6 -parser SIMPLE -inFile ins.csv -outFile outm.csv -man -may -mc 10 -mSn -mt -ps -pt 6 -sxw 1 -targets ASFIS -xml -xslTemplate csv + String execution = "java -Xmx512m -Xmx1024m -jar " + pathToComet + "SpeciMEn1.0.71.jar -parser " + parser + " -inFile " + inFile + " " + "-outFile " + outFile + " " + "-man " + "-may " + "-mc "+maxresults+" -mSn " + "-mt " + "-ps " + "-pt 6" + " -sxw " + sxnw + " " + "-targets " + reference + " " + "-xml -xslTemplate csv"; + System.out.println("Executing: " + execution); + Process process = null; + try{ + process = Runtime.getRuntime().exec(execution); + + BufferedReader br = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String line = br.readLine(); + System.out.println(line); + while (line != null) { + line = br.readLine(); + System.out.println(line); + } + }catch(Exception e){ + System.out.println("Unable to execute the program"); + throw e; + } + finally{ + if (process!=null) + process.destroy(); + } + } + + public static List> parseCometParserOutput(String parserOutput) throws Exception { + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(parserOutput)), "UTF-8")); + String line = br.readLine(); // skip headers + line = br.readLine(); + List> parsednames = new ArrayList>(); + while (line != null) { + System.out.println("reading from parser output: "+line); + if (line.trim().length() > 0) { + List tokens = Transformations.parseCVSString(line, ";"); + int tokenslength = tokens.size(); + // take the 3rd and 4th elements: PARSED_SCIENTIFIC_NAME;PARSED_AUTHORITY + + String scientificname = ""; + if (tokenslength > 2) + scientificname = tokens.get(2).replace(",", "").trim(); + + String author = ""; + if (tokenslength > 3) + author = tokens.get(3).replace(",", "").trim(); + + if (scientificname.length()==0 && author.length()==0) + scientificname = line.replace(",", ""); + + Tuple t = new Tuple(scientificname, author); + parsednames.add(t); + } + + line = br.readLine(); + } + br.close(); + + return parsednames; + } + + // puts also in normal format, e.g. Species Abra alba (W. Wood, 1802) + + List scores = new ArrayList(); + List matchednames= new ArrayList(); + public List getScores(){ + return scores; + } + + public List getMatches(){ + return matchednames; + } + + public List> parseCometOutput(String parserOutput) throws Exception { + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(parserOutput)), "UTF-8")); + String line = br.readLine(); // skip headers + line = br.readLine(); + List> parsednames = new ArrayList>(); + while (line != null) { + + if (line.trim().length() > 0) { + System.out.println("Processing Comet output. Line: "+line); + List tokens = Transformations.parseCVSString(line, ","); + int tokenslength = tokens.size(); + + String score = ""; + if (tokenslength > 4) + score = tokens.get(4).trim(); + + String scientificname = ""; + if (tokenslength > 7) + scientificname = tokens.get(7).trim(); + + String author = ""; + if (tokenslength > 8) + author = tokens.get(8).replace("(","").replace(")", "").trim(); + + String matched = scientificname.replace("(","").replace(")", "").replace(",", "").replace(";", ""); + parsednames.add(new Tuple(matched,author,score)); + } + line = br.readLine(); + } + br.close(); + return parsednames; + } + + public static void dumpCometInput(String inputFile, List> rawNames) throws Exception { + BufferedWriter bw = new BufferedWriter(new FileWriter(new File(inputFile))); + int size = rawNames.size(); + for (int i = 0; i < size; i++) { + System.out.println("Writing the following to file: "+rawNames.get(i)); + Tuple t = rawNames.get(i); + String author = ""; + if (t.getElements().get(1).length()>0) + author = " ("+t.getElements().get(1)+")"; + + bw.append(t.getElements().get(0)+ author) ; + if (i < (size - 1)) + bw.append("\n"); + } + + bw.close(); + } + + public List> match(String parser, String reference, String sandboxFolder, List> inputNamesList, float soundexweightF, int maxResults) throws Exception{ + File FmatcherinputFile = new File(sandboxFolder,"inputCometMatcher.csv"); + File FmatcheroutputFile = new File(sandboxFolder,"outputCometMatcher.csv"); + try{ + FmatcherinputFile.delete(); + }catch(Exception e){} + + try{ + FmatcheroutputFile.delete(); + }catch(Exception e){} + + String matcherinputFile = FmatcherinputFile.getAbsolutePath(); + String matcheroutputFile = FmatcheroutputFile.getAbsolutePath(); + CometMatcherManager.dumpCometInput(matcherinputFile, inputNamesList); + CometMatcherManager.cometMatch(sandboxFolder,parser, reference, matcheroutputFile, matcherinputFile, soundexweightF, maxResults); + List> outputNames = parseCometOutput(matcheroutputFile); + return outputNames; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/EVBPreprocessing.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/EVBPreprocessing.java new file mode 100644 index 0000000..486e892 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/EVBPreprocessing.java @@ -0,0 +1,162 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.utils.Tuple; + +public class EVBPreprocessing { + + +// "preparsecleaning";"Pre-parsing cleaning";"Does not require knowledge of where the individual components of the complete namestring start or end" + public static String[][] preparsecleaning = { + {" ", " ","3"},//space + {""+(char)10," ","3"}, //space + {" {2,}"," ","3"}, //consecutive spaces + {"^ ", "","3"},//leading space + {" $", "","3"},//trailing space + {"[?]", "", "1"},//uncertain identification + {" v(ar)?\\.? "," v. ","1" },//standardise variety indication + {" f(orm(a)?)?.? ", " f. ","1"}//standardise form indication + }; + + public static String[][] postparsecleaning = { + {" sp[\\.]?( ?[1-9a-zA-Z])?$","","1"}//remove temporary species indication + }; + + public static String[] preparsecleaningorigins = { + " ", //space + ""+(char)10, //space + " {2,}", //consecutive spaces + "^ ", //leading space + " $", //trailing space + "[?]", //uncertain identification + " v(ar)?\\.? ", //standardise variety indication + " f(orm(a)?)?.? " //standardise form indication + }; + + public static String[] preparsecleaningtargets = { + " ", //space + " ", //space + " ", //consecutive spaces + "", //leading space + "", //trailing space + "", //uncertain identification + " v. ", //standardise variety indication + " f. " //standardise form indication + }; + + //"postparsecleaning";"Post-parsing cleaning";"Does require knowledge of where the individual components of the complete namestring start or end; assumes namestring is split in name proper and authority" + public static String[] postparsecleaningorigin = { + " sp[\\.]?( ?[1-9a-zA-Z])?$" //remove temporary species indication + }; + + public static String[] postparsecleaningtargets = { + "" //remove temporary species indication + }; + + //"fuzzymatch";"Fuzzy matching";"Based on original idea from Tony Rees" + public static String[] fuzzymatchorigins = { + "h", //remove all characters h + "y", //all y to i + "s|k" //all s and k to c + }; + + public static String[] fuzzymatchtargets = { + "''", //remove all characters h + "i", //all y to i + "c" //all s and k to c + }; + + public static enum Preprocessors{ + EXPERT_RULES, + NONE + } + + public static boolean appliesToScientificName(int regexProperty){ + int t = (1 & regexProperty); + return t>0; + } + + public static boolean appliesToAuthorship(int regexProperty){ + int t = (2 & regexProperty); + return t>0; + } + + public static List> populateTuples(List rawnames){ + List> preprocessednames = new ArrayList>(); + for (String rawn:rawnames){ + preprocessednames.add(new Tuple(rawn,"")); + } + return preprocessednames; + } + + public List> preprocess(String parser, String sandboxFolder, List rawnamesFiltered) throws Exception{ + File FParserinputFile = new File(sandboxFolder,"inputEVBParser.csv"); + File FParseroutputFile = new File(sandboxFolder,"outputEVBParser.csv"); + + try{ + FParserinputFile.delete(); + }catch(Exception e){ + + } + + try{ + FParseroutputFile.delete(); + }catch(Exception e){ + + } + + String parserinputFile = FParserinputFile.getAbsolutePath(); + String parseroutputFile = FParseroutputFile.getAbsolutePath(); + + List preprocessedrawnames = new ArrayList(rawnamesFiltered); + //apply evb preprocess + int namessize = preprocessedrawnames.size(); + + System.out.println("Applying preprocessing to strings"); + + for (int i=0;i"+preparsecleaningtargets[j]+"="+preprocessedrawname); + } + } + + System.out.println("Applying parsing"); + + CometMatcherManager.dumpCometInput(parserinputFile, populateTuples(preprocessedrawnames)); + CometMatcherManager.cometParse(sandboxFolder,parser, parseroutputFile, parserinputFile); + List> parsedNames = CometMatcherManager.parseCometParserOutput(parseroutputFile); + + int pnamessize = parsedNames.size(); + System.out.println("Applying postprocessing to "+pnamessize+" strings"); + for (int i=0;i normalized = CometMatcherManager.parseCometParserOutput("./PARALLEL_PROCESSING/outs.csv"); + for (String norm:normalized){ + System.out.println(norm); + } + */ + System.out.println("Gadus var. morhua".replaceAll(" v(ar)?\\.? ", " v. ")); + + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/GenericWorkflow.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/GenericWorkflow.java new file mode 100644 index 0000000..18721e7 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/GenericWorkflow.java @@ -0,0 +1,77 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Parser; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.PostProcessor; + +public abstract class GenericWorkflow { + + protected HashMap globalparameters; + protected String sandboxFolder; + + protected Parser parser; + protected List matchersList; + protected PostProcessor postprocessor; + + + public List getMatchersList(){ + return matchersList; + } + + public void init(HashMap globalparameters){ + this.globalparameters=globalparameters; + } + + public GenericWorkflow(String sandboxFolder, HashMap globalparameters){ + this.globalparameters = globalparameters; + this.sandboxFolder = sandboxFolder; + matchersList = new ArrayList(); + init(globalparameters); + } + + //merges without repetitions + protected abstract MatcherOutput mergeOutputs(List outslist); + + public MatcherOutput executeChainedWorkflow(List rawentries) throws Exception{ + //Preparsing and parsing + long t0 = System.currentTimeMillis(); + parser.init(sandboxFolder, globalparameters); + System.out.println("Parsing.."); + MatcherInput currentInput = parser.parse(rawentries); + System.out.println("Parsed - Time: "+(System.currentTimeMillis()-t0)); + List outputs = new ArrayList(); + System.out.println("Matching.."); + //matching + for (Matcher matcher:matchersList){ + MatcherOutput currentoutput = new MatcherOutput(); + if (currentoutput!=null){ + long t1 = System.currentTimeMillis(); + System.out.println("Matching with .."+matcher); + currentoutput = matcher.match(currentInput); + outputs.add(currentoutput); + System.out.println("Matched - Time: "+(System.currentTimeMillis()-t1)); + } + } + System.out.println("Merging.."); + long t2=System.currentTimeMillis(); + MatcherOutput finaloutput = mergeOutputs(outputs); + System.out.println("Merged - Time: "+(System.currentTimeMillis()-t2)); + if (postprocessor!=null){ + long t3 = System.currentTimeMillis(); + System.out.println("Postprocessing.."); + finaloutput = postprocessor.postprocessMatches(finaloutput); + System.out.println("Postprocessed - Time: "+(System.currentTimeMillis()-t3)); + } + System.out.println("END."); + return finaloutput; + } + + public void resetMatchers(List newmatchers){ + this.matchersList = null; + this.matchersList=newmatchers; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/MatcherInput.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/MatcherInput.java new file mode 100644 index 0000000..24df0aa --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/MatcherInput.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts; + +import java.util.ArrayList; +import java.util.List; + + +public class MatcherInput { + + + public List entries = new ArrayList(); + + public MatcherInput(){ + + } + + public void addEntry(int id, String originalName, String parsedScientificName,String parsedAuthorship, List otherElements){ + entries.add(new SingleEntry(id, originalName, parsedScientificName, parsedAuthorship, otherElements)); + } + + public int getEntriesNumber(){ + return entries.size(); + } + + public SingleEntry getEntry(int index){ + return entries.get(index); + } + +} + diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/MatcherOutput.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/MatcherOutput.java new file mode 100644 index 0000000..c3ff106 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/MatcherOutput.java @@ -0,0 +1,40 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts; + +import java.util.ArrayList; +import java.util.List; + + +public class MatcherOutput { + + public List entries = new ArrayList(); + + public MatcherOutput(){ + } + + public void addEntry(int id, String originalName, String parsedScientificName,String parsedAuthorship, double matchingScore, String targetDataSource, String targetID, String targetScientificName, String targetAuthor, List otherElements){ + entries.add(new SingleEntry(id, originalName, parsedScientificName,parsedAuthorship, matchingScore, targetDataSource, targetID, targetScientificName, targetAuthor, otherElements)); + } + + public int getEntriesNumber(){ + return entries.size(); + } + + public SingleEntry getEntry(int index){ + return entries.get(index); + } + + public boolean contains(String inputName, String scientificName, String author,String targetID){ + for (SingleEntry entry: entries){ + if (entry.originalName.equals(inputName) + && + (entry.targetScientificName.equals(scientificName)) + && + (entry.targetAuthor.equals(author)) + && + (entry.targetID.equals(targetID))) + return true; + } + return false; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/RealTimeMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/RealTimeMatcher.java new file mode 100644 index 0000000..f350689 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/RealTimeMatcher.java @@ -0,0 +1,238 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; + +import org.gcube.contentmanagement.graphtools.utils.MathFunctions; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanagement.lexicalmatcher.utils.DistanceCalculator; +import org.gcube.contentmanagement.lexicalmatcher.utils.FileTools; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; + +public class RealTimeMatcher { + + public String PARSER; + public String INPUT_DATA_SOURCE_ID; + public String INPUT_DATA_ID; + public String INPUT_DATA; + public String PREPARSED_INPUT_DATA; + public String PARSED_SCIENTIFIC_NAME; + public String PARSED_AUTHORITY; + public String POST_PARSED_SCIENTIFIC_NAME; + public String POST_PARSED_AUTHORITY; + static HashMap> officialTafsMap; + static int cachCleaningTime = 2 * 60 * 60 *1000; + static String outputheaders = "SOURCE_DATASOURCE_ID;SOURCE_ID;SOURCE_DATA;PRE_PARSED_SOURCE_DATA;PARSED_SCIENTIFIC_NAME;PARSED_AUTHORITY;PARSER;POST_PARSED_SCIENTIFIC_NAME;POST_PARSED_AUTHORITY;MATCHING_SCORE;TARGET_DATA_SOURCE;TARGET_DATA_ID;TARGET_DATA_SCIENTIFIC_NAME;TARGET_DATA_AUTHORITY;TARGET_DATA_KINGDOM;TARGET_DATA_PHYLUM;TARGET_DATA_CLASS;TARGET_DATA_ORDER;TARGET_DATA_FAMILY;TARGET_DATA_GENUS;TARGET_DATA_SPECIES;TARGET_DATA_VERNACULAR_NAMES"; + + // MATCHING_SCORE TARGET_DATA_SOURCE TARGET_DATA_ID TARGET_DATA_SCIENTIFIC_NAME TARGET_DATA_AUTHORITY TARGET_DATA_KINGDOM TARGET_DATA_PHYLUM TARGET_DATA_CLASS TARGET_DATA_ORDER TARGET_DATA_FAMILY TARGET_DATA_GENUS TARGET_DATA_SPECIES TARGET_DATA_VERNACULAR_NAMES + public RealTimeMatcher(){ + //TODO: scheduler +// databasecheckScheduler = new Timer(); databasecheckScheduler.schedule(new DatabaseController(), 0, refreshTime); + } + + private class TafsCacheCleaner extends TimerTask { + @Override + public void run() { + AnalysisLogger.getLogger().debug("RealTimeMatcher: Cache cleaned"); + officialTafsMap = null; + System.gc(); + } + + } + + class TafInfo { + double MATCHING_SCORE ; + String TARGET_DATA_SOURCE ; + String TARGET_DATA_ID; + String TARGET_DATA_SCIENTIFIC_NAME; + String TARGET_DATA_AUTHORITY; + String TARGET_DATA_KINGDOM; + String TARGET_DATA_PHYLUM; + String TARGET_DATA_CLASS; + String TARGET_DATA_ORDER; + String TARGET_DATA_FAMILY; + String TARGET_DATA_GENUS; + String TARGET_DATA_SPECIES; + String TARGET_DATA_VERNACULAR_NAMES; + } + + + + private HashMap getCurrentTaf(String tafFile) throws Exception { + try { + String file = FileTools.loadString(tafFile, "UTF-8"); + String[] tafrows = file.split("\n"); + HashMap tafMap; + if (officialTafsMap==null){ + officialTafsMap = new HashMap>(); + Timer cachecleaner = new Timer(); + cachecleaner.schedule(new TafsCacheCleaner(), cachCleaningTime); + } + if (officialTafsMap.get(tafFile)!=null) + return officialTafsMap.get(tafFile); + + tafMap = new HashMap(); + + for (String row : tafrows) { + List elements = Transformations.parseCVSString(row, "\t"); + TafInfo tafInfo = new TafInfo(); + tafInfo.TARGET_DATA_ID = elements.get(0); + tafInfo.TARGET_DATA_SCIENTIFIC_NAME = elements.get(37); + tafInfo.TARGET_DATA_AUTHORITY = elements.get(41); + tafInfo.TARGET_DATA_KINGDOM = elements.get(1); + tafInfo.TARGET_DATA_PHYLUM = elements.get(5); + tafInfo.TARGET_DATA_CLASS = elements.get(9); + tafInfo.TARGET_DATA_ORDER = elements.get(13); + tafInfo.TARGET_DATA_FAMILY = elements.get(17); + tafInfo.TARGET_DATA_GENUS = elements.get(21); + tafInfo.TARGET_DATA_SPECIES = elements.get(29); + tafInfo.TARGET_DATA_VERNACULAR_NAMES = ""; + + tafMap.put((tafInfo.TARGET_DATA_SCIENTIFIC_NAME + " " + tafInfo.TARGET_DATA_AUTHORITY).toLowerCase(), tafInfo); + } + + officialTafsMap.put(tafFile, tafMap); + return tafMap; + } catch (Exception e) { + e.printStackTrace(); + throw new Exception("Error: could not read Taf file"); + } + } + + private void getCurrentInput(String postParserFile) throws Exception { + try { + String file = FileTools.loadString(postParserFile, "UTF-8"); + + String row = file.split("\n")[1]; + List elements = Transformations.parseCVSString(row, ";"); + PARSER = elements.get(0); + INPUT_DATA_SOURCE_ID = elements.get(1); + INPUT_DATA_ID = elements.get(2); + INPUT_DATA = elements.get(3); + PREPARSED_INPUT_DATA = elements.get(4); + PARSED_SCIENTIFIC_NAME = elements.get(5); + PARSED_AUTHORITY = elements.get(6); + POST_PARSED_SCIENTIFIC_NAME = elements.get(7); + POST_PARSED_AUTHORITY = elements.get(8); + } catch (Exception e) { + throw new Exception("Error: could not read post-parsing file"); + } + } + + public void match(String taffile, String tafName, String postParserFile, String outputFile, double threshold, int maxvalues) throws Exception { + + AnalysisLogger.getLogger().debug("RealTimeMatcher started with the following parameters: "); + AnalysisLogger.getLogger().debug("taffile: "+taffile); + AnalysisLogger.getLogger().debug("tafName: "+tafName); + AnalysisLogger.getLogger().debug("postParserFile: "+postParserFile); + AnalysisLogger.getLogger().debug("outputFile: "+outputFile); + AnalysisLogger.getLogger().debug("threshold: "+threshold); + AnalysisLogger.getLogger().debug("maxvalues: "+maxvalues); + long t0 = System.currentTimeMillis(); + getCurrentInput(postParserFile); + long t1 = System.currentTimeMillis(); + AnalysisLogger.getLogger().debug("Current Input rebuilt in "+(t1-t0)); + + /* + Properties p = PropertyLoader.loadProperties("cache/cache.ccf", ClassLoader.getSystemClassLoader()); + CompositeCacheManager ccm = CompositeCacheManager.getUnconfiguredInstance(); + ccm.configure(p); + CacheAccess cache = JCS.getInstance("zone"); + cache.put("hello", "world"); + System.out.println("Retrieved: "+cache.get("hello")); + */ + + HashMap scientificnames = getCurrentTaf(taffile); +// TafInfo info = scientificnames.get((POST_PARSED_SCIENTIFIC_NAME + " " + POST_PARSED_AUTHORITY).toLowerCase()); + long t2 = System.currentTimeMillis(); + AnalysisLogger.getLogger().debug("TAF rebuilt in "+(t2-t1)); + /* + if (info != null) { + AnalysisLogger.getLogger().debug(info.TARGET_DATA_SCIENTIFIC_NAME+" "+info.TARGET_DATA_AUTHORITY); + return; + } + */ + + DistanceCalculator dc = new DistanceCalculator(); + List bestTafs= new ArrayList(); + int bestTafList = 0; + for (TafInfo testInfo:scientificnames.values()){ + double snameScore = dc.CD(false, POST_PARSED_SCIENTIFIC_NAME, testInfo.TARGET_DATA_SCIENTIFIC_NAME, true, false); + double authScore = 0.5; + if (POST_PARSED_AUTHORITY!=null && testInfo.TARGET_DATA_AUTHORITY!=null && POST_PARSED_AUTHORITY.length()>0 && testInfo.TARGET_DATA_AUTHORITY.length()>0) + authScore=dc.CD(false, POST_PARSED_AUTHORITY, testInfo.TARGET_DATA_AUTHORITY, true, false); + else if (POST_PARSED_AUTHORITY==null || POST_PARSED_AUTHORITY.length() == 0 ) + authScore=1; + if (authScore*snameScore>threshold){ + testInfo.MATCHING_SCORE=(authScore*snameScore); + insertTaf(bestTafs,testInfo,bestTafList); + bestTafList++; + } + } + + long t3 = System.currentTimeMillis(); + AnalysisLogger.getLogger().debug("Scientific Names rebuilt in "+(t3-t2)); + + AnalysisLogger.getLogger().debug("Results"); + int belements = 0; + + BufferedWriter bw = new BufferedWriter(new FileWriter(new File(outputFile))); + bw.write(outputheaders+"\n"); + // static String outputheaders = "SOURCE_DATASOURCE_ID;SOURCE_ID;SOURCE_DATA;PRE_PARSED_SOURCE_DATA;PARSED_SCIENTIFIC_NAME;PARSED_AUTHORITY;PARSER;POST_PARSED_SCIENTIFIC_NAME;POST_PARSED_AUTHORITY;MATCHING_SCORE;TARGET_DATA_SOURCE;TARGET_DATA_ID;TARGET_DATA_SCIENTIFIC_NAME;TARGET_DATA_AUTHORITY;TARGET_DATA_KINGDOM;TARGET_DATA_PHYLUM;TARGET_DATA_CLASS;TARGET_DATA_ORDER;TARGET_DATA_FAMILY;TARGET_DATA_GENUS;TARGET_DATA_SPECIES;TARGET_DATA_VERNACULAR_NAMES"; + for (TafInfo b:bestTafs){ + AnalysisLogger.getLogger().debug(b.TARGET_DATA_SCIENTIFIC_NAME+" "+b.TARGET_DATA_AUTHORITY+" "+b.MATCHING_SCORE); + bw.write(q(INPUT_DATA_SOURCE_ID)+";"+q(INPUT_DATA_ID)+";"+ + q(INPUT_DATA)+";"+q(PREPARSED_INPUT_DATA)+";"+q(PARSED_SCIENTIFIC_NAME)+";"+ + q(PARSED_AUTHORITY)+";"+q(PARSER)+";"+q(POST_PARSED_SCIENTIFIC_NAME)+";"+ + q(POST_PARSED_AUTHORITY)+";"+MathFunctions.roundDecimal(b.MATCHING_SCORE,2)+";"+ + q(tafName)+";"+ + q(b.TARGET_DATA_ID)+";"+q(b.TARGET_DATA_SCIENTIFIC_NAME)+";"+q(b.TARGET_DATA_AUTHORITY)+";"+ + q(b.TARGET_DATA_KINGDOM)+";"+q(b.TARGET_DATA_PHYLUM)+";"+q(b.TARGET_DATA_CLASS)+";"+ + q(b.TARGET_DATA_ORDER)+";"+q(b.TARGET_DATA_FAMILY)+";"+q(b.TARGET_DATA_GENUS)+";"+ + q(b.TARGET_DATA_SPECIES)+";"+q(b.TARGET_DATA_VERNACULAR_NAMES)+ + "\n"); + + if (belements>maxvalues) + break; + + belements++; + } + + bw.close(); + } + + private String q(String in){ + return "\""+in+"\""; + } + + private void insertTaf(List elements, TafInfo toInsert, int elementsLen){ + + int counter = 0; + for (TafInfo e:elements){ + if (e.MATCHING_SCORE otherElements; + + public SingleEntry(int id, String originalName, String parsedScientificName,String parsedAuthorship, double matchingScore, String targetDataSource, String targetID, String targetScientificName, String targetAuthor, List otherElements){ + this.id=id; + this.originalName=originalName; + this.parsedScientificName=parsedScientificName; + this.parsedAuthorship=parsedAuthorship; + this.otherElements=otherElements; + this.matchingScore=matchingScore; + this.targetDataSource=targetDataSource; + this.targetID=targetID; + this.targetScientificName=targetScientificName; + this.targetAuthor=targetAuthor; + } + + public SingleEntry(int id, String originalName, String parsedScientificName,String parsedAuthorship, List otherElements){ + this(id,originalName,parsedScientificName,parsedAuthorship,0,"","","","",otherElements); + } + + @Override + public String toString(){ + return "\""+id+"\";\""+originalName+"\";\""+targetID+"\";\""+targetScientificName+"\";\""+targetAuthor+"\";\""+matchingScore+"\""; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/YasmeenMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/YasmeenMatcher.java new file mode 100644 index 0000000..008afb0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/abstracts/YasmeenMatcher.java @@ -0,0 +1,164 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts; + +import java.io.File; +import java.security.Permission; +import java.util.HashMap; +import java.util.UUID; + +import org.apache.tools.ant.ExitException; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.LevensteinMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenFileTools; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public abstract class YasmeenMatcher implements Matcher { + + public static String matcherLib = "YASMEEN-matcher-1.2.0.1.jar"; + protected String sandBoxFolder = "./"; + public double threshold = 0.4; + public int maxResults = 10; + public HashMap parameters; + + HashMap urlsmap = createReferenceMap(); + static HashMap performancemap = createPerformanceMap(); + + protected static HashMap createReferenceMap() { + HashMap urlsMaps = new HashMap(); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.ASFIS.name(), "http://goo.gl/qfeTp7"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.FISHBASE.name(), "http://goo.gl/FcnUc0"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS.name(), "http://goo.gl/AUcUXt"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_ANIMALIA.name(), "http://goo.gl/mW1fvb"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_CNIDARIA.name(), "http://goo.gl/N6vCq8"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_ECHINODERMATA.name(), "http://goo.gl/2qQThI"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_PLATYHELMINTHES.name(), "http://goo.gl/RLmCn2"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.COL_FULL.name(), "http://goo.gl/fJ9feg"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.COL_CHORDATA.name(), "http://goo.gl/11upZC"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.COL_MAMMALIA.name(), "http://goo.gl/J4fga6"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.IRMNG_ACTINOPTERYGII.name(), "http://goo.gl/Z8eRly"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.WORMS_ANIMALIA.name(), "http://goo.gl/XRMWgr"); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name(), "http://goo.gl/OdPRiA"); + + return urlsMaps; + } + + protected HashMap createReferenceMapFile(String localFolder) { + HashMap urlsMaps = new HashMap(); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.ASFIS.name(), new File(localFolder, "ASFIS_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.FISHBASE.name(), new File(localFolder, "FISHBASE_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS.name(), new File(localFolder, "OBIS_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_ANIMALIA.name(), new File(localFolder, "OBIS_Animalia_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_CNIDARIA.name(), new File(localFolder, "OBIS_Cnidaria_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_ECHINODERMATA.name(), new File(localFolder, "OBIS_Echinodermata_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.OBIS_PLATYHELMINTHES.name(), new File(localFolder, "OBIS_Platyhelminthes_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.COL_FULL.name(), new File(localFolder, "COL_FULL_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.COL_CHORDATA.name(), new File(localFolder, "COL_Chordata_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.COL_MAMMALIA.name(), new File(localFolder, "COL_Mammalia_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.IRMNG_ACTINOPTERYGII.name(), new File(localFolder, "IRMNG_Actinopterygii_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.WORMS_ANIMALIA.name(), new File(localFolder, "WORMS_Animalia_taxa.taf").getAbsolutePath()); + urlsMaps.put(YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name(), new File(localFolder, "WoRMS_Pisces_taxa.taf").getAbsolutePath()); + + return urlsMaps; + } + + protected static HashMap createPerformanceMap() { + HashMap performanceMap = new HashMap(); + performanceMap.put(YasmeenGlobalParameters.Performance.MAX_ACCURACY.name(), ""); + performanceMap.put(YasmeenGlobalParameters.Performance.LOW_SPEED.name(), "-dpt 10"); + performanceMap.put(YasmeenGlobalParameters.Performance.MEDIUM_SPEED.name(), "-dpt 6"); + performanceMap.put(YasmeenGlobalParameters.Performance.HIGH_SPEED.name(), "-dpt 2"); + performanceMap.put(YasmeenGlobalParameters.Performance.MAX_SPEED.name(), "-dpt 0"); + return performanceMap; + } + + protected abstract String getMatchlets(); + + protected abstract String getLexicalDistancesWeights(); + + protected abstract String getStemming(); + + public YasmeenMatcher(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + init(sandboxfolder, threshold, maxResults, parameters); + } + + @Override + public void init(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + this.sandBoxFolder = sandboxfolder; + this.parameters = parameters; + this.threshold = threshold; + this.maxResults = maxResults; + } + + @Override + public MatcherOutput match(MatcherInput inputName) throws Exception { + + String inputFileName = parameters.get(YasmeenGlobalParameters.parserOutputFileParam); + String uuid = ("" + UUID.randomUUID()).replace("-", ""); + String outputFileName = "matcherOutput" + uuid + ".csv"; + File outfile = new File(sandBoxFolder, outputFileName); + if (outfile.exists()) { + outfile.delete(); + } + + // String outputFileName = parameters.get(YasmeenGlobalParameters.parserOutputFileParam); + String taxaAuthorityFile = parameters.get(YasmeenGlobalParameters.taxaAuthorityFileParam); + String performanceMap = parameters.get(YasmeenGlobalParameters.performanceParam); + + String performanceCommand = performancemap.get(performanceMap); + if (performanceCommand == null) + performanceCommand = ""; + + boolean stemmit = Boolean.parseBoolean(parameters.get(YasmeenGlobalParameters.useStemmedGenusAndSpecies)); + String normalizeMatchlets = ""; + String standardMatchlets = ""; + if (stemmit) + normalizeMatchlets = getStemming(); + else + standardMatchlets = getMatchlets(); + + /* ENABLE LOCAL FILES USAGE */ + /* + * String localFilesFolder = parameters.get(YasmeenGlobalParameters.staticFilesFolderParam); if (localFilesFolder==null){ localFilesFolder=""; urlsmap=createReferenceMap(); } else urlsmap=createReferenceMapFile(localFilesFolder); + */ + // java -jar YASMEEN-matcher-1.1.1.jar -inFile output.csv -refData ASFIS@https://dl.dropboxusercontent.com/u/12809149/ASFIS_taxa.taf.gz -outFile matchout.csv -man -may -mSn -mt -mc 10 -ps -pt 1.5x -xml -xslTemplate csv -law 90:30:60 + // YasmeenFileTools.writeYasmeenMatcherInput(sandBoxFolder, inputFileName, parser, inputName); + + // org.fao.fi.comet.domain.species.tools.process.matching.cli.MatchingEngine + + String localFilesFolder = parameters.get(YasmeenGlobalParameters.staticFilesFolderParam); + if ((localFilesFolder != null) && (localFilesFolder.length() > 0) + && (taxaAuthorityFile.equals(YasmeenGlobalParameters.BuiltinDataSources.FISHBASE.name())) + && (this instanceof LevensteinMatcher) + && new File(createReferenceMapFile(localFilesFolder).get(taxaAuthorityFile)).exists()) { + RealTimeMatcher rm = new RealTimeMatcher(); + HashMap filesMap = createReferenceMapFile(localFilesFolder); + rm.match(filesMap.get(taxaAuthorityFile), taxaAuthorityFile, new File(sandBoxFolder, inputFileName).getAbsolutePath(), outfile.getAbsolutePath(), threshold, maxResults); + } + else { + String execution = "java -Xmx512m -Xmx1024m -jar " + new File(sandBoxFolder, matcherLib).getAbsolutePath() + " -inFile " + new File(sandBoxFolder, inputFileName).getAbsolutePath() + " -refData " + taxaAuthorityFile + "@" + urlsmap.get(taxaAuthorityFile) + " -outFile " + outfile.getAbsolutePath() + " " + normalizeMatchlets + " -mc " + maxResults + " -mst " + threshold + " -pt 1x -xml -xslTemplate csv" + " " + getLexicalDistancesWeights() + " " + standardMatchlets + " " + performanceCommand; + AnalysisLogger.getLogger().debug(execution); + YasmeenFileTools.callYasmeen(execution.trim()); + } + MatcherOutput mo = YasmeenFileTools.getYasmeenMatcherOutput(sandBoxFolder, outputFileName); + outfile.delete(); + return mo; + } + + private static class NoExitSecurityManager extends SecurityManager { + @Override + public void checkPermission(Permission perm) { + // allow anything. + } + + @Override + public void checkPermission(Permission perm, Object context) { + // allow anything. + } + + @Override + public void checkExit(int status) { + super.checkExit(status); + throw new ExitException(status); + } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/FuzzyMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/FuzzyMatcher.java new file mode 100644 index 0000000..c075450 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/FuzzyMatcher.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers; + +import java.util.HashMap; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; + + +public class FuzzyMatcher extends YasmeenMatcher{ + + public FuzzyMatcher(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + super(sandboxfolder,threshold, maxResults, parameters); + } + + @Override + protected String getMatchlets() { + + return "-mftm"; + } + + @Override + protected String getLexicalDistancesWeights() { + return ""; + } + + protected String getStemming(){ + return "-mftm"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/GsayMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/GsayMatcher.java new file mode 100644 index 0000000..9c26b16 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/GsayMatcher.java @@ -0,0 +1,30 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers; + +import java.util.HashMap; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; + + +public class GsayMatcher extends YasmeenMatcher{ + + public GsayMatcher(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + super(sandboxfolder,threshold, maxResults,parameters); + } + + @Override + protected String getMatchlets() { + + return "-mgsay"; + } + + @Override + protected String getLexicalDistancesWeights() { + return ""; + } + + @Override + protected String getStemming() { + return "-mgsay"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/LevensteinMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/LevensteinMatcher.java new file mode 100644 index 0000000..6009b08 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/LevensteinMatcher.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers; + +import java.util.HashMap; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; + + +public class LevensteinMatcher extends YasmeenMatcher { + + public LevensteinMatcher(String sandboxfolder, double threshold, int maxResults,HashMap parameters) { + super(sandboxfolder, threshold, maxResults,parameters); + } + + @Override + protected String getMatchlets() { + return "-mSn -man -may -mSnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + + @Override + protected String getLexicalDistancesWeights() { + return "-law 100:0:0"; + } + + @Override + protected String getStemming() { + return "-mNgn -mNsn -man -may -mNgnt 0.0001 -mNsnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/MixedLexicalMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/MixedLexicalMatcher.java new file mode 100644 index 0000000..da36829 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/MixedLexicalMatcher.java @@ -0,0 +1,28 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers; + +import java.util.HashMap; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; + + +public class MixedLexicalMatcher extends YasmeenMatcher{ + + public MixedLexicalMatcher(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + super(sandboxfolder, threshold, maxResults,parameters); + } + + @Override + protected String getMatchlets() { + return "-mSn -man -may -mSnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + + @Override + protected String getLexicalDistancesWeights() { + return "-law 100:100:100"; + } + + @Override + protected String getStemming() { + return "-mNgn -mNsn -man -may -mNgnt 0.0001 -mNsnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/SoundexMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/SoundexMatcher.java new file mode 100644 index 0000000..be2e2e2 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/SoundexMatcher.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers; + +import java.util.HashMap; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; + + +public class SoundexMatcher extends YasmeenMatcher{ + + public SoundexMatcher(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + super(sandboxfolder, threshold, maxResults,parameters); + } + + @Override + protected String getMatchlets() { + return "-mSn -man -may -mSnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + + @Override + protected String getLexicalDistancesWeights() { + return "-law 0:100:0"; + } + + @Override + protected String getStemming() { + return "-mNgn -mNsn -man -may -mNgnt 0.0001 -mNsnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/TrigramMatcher.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/TrigramMatcher.java new file mode 100644 index 0000000..f3863f4 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/matchers/TrigramMatcher.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers; + +import java.util.HashMap; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.YasmeenMatcher; + + +public class TrigramMatcher extends YasmeenMatcher{ + + public TrigramMatcher(String sandboxfolder, double threshold, int maxResults, HashMap parameters) { + super(sandboxfolder, threshold, maxResults,parameters); + } + + @Override + protected String getMatchlets() { + return "-mSn -man -may -mSnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + + @Override + protected String getLexicalDistancesWeights() { + return "-law 0:0:100"; + } + + @Override + protected String getStemming() { + return "-mNgn -mNsn -man -may -mNgnt 0.0001 -mNsnt 0.0001 -mant 0.0001 -mayt 0.0001"; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/parsers/YasmeenParser.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/parsers/YasmeenParser.java new file mode 100644 index 0000000..9b27e4c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/parsers/YasmeenParser.java @@ -0,0 +1,48 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.parsers; + +import java.io.File; +import java.util.HashMap; +import java.util.List; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherInput; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Parser; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenFileTools; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + + +public class YasmeenParser implements Parser{ + + public static String parserLib = "YASMEEN-parser-1.2.0.jar"; + + protected String sandboxFolder = "./"; + public HashMap parameters; + + @Override + public void init(String sandboxfolder, HashMap parameters) { + this.sandboxFolder = sandboxfolder; + this.parameters = parameters; + } + + @Override + public MatcherInput parse (List rawnames) throws Exception{ + String parserName = parameters.get(YasmeenGlobalParameters.parserNameParam); + String inputFileName = parameters.get(YasmeenGlobalParameters.parserInputFileParam); + String outputFileName = parameters.get(YasmeenGlobalParameters.parserOutputFileParam); + Boolean preparsing = Boolean.valueOf(parameters.get(YasmeenGlobalParameters.activatePreParsingProcessing)); + + YasmeenFileTools.writeYasmeenParserInput(sandboxFolder, inputFileName, rawnames); + + if (parserName==null || parserName.equalsIgnoreCase(YasmeenGlobalParameters.BuiltinParsers.NONE.name()) ) + parserName = "IDENTITY"; + + String execution = "java -Xmx512m -Xmx1024m -jar "+new File(sandboxFolder,parserLib)+" -inFile "+new File(sandboxFolder,inputFileName).getAbsolutePath()+" -outFile "+new File(sandboxFolder,outputFileName).getAbsolutePath()+" -parser "+parserName; + if (preparsing) + execution+= " -preParsingRuleset commonPreparsingRules -preParsingRuleset bionymPreparsingRules -postParsingRuleset bionymPostparsingRules"; + + YasmeenFileTools.callYasmeen(execution); + MatcherInput input = YasmeenFileTools.getYasmeenParserOutput(sandboxFolder, outputFileName); + + return input; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/workflows/BiOnymWF.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/workflows/BiOnymWF.java new file mode 100644 index 0000000..61f60e6 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/implementations/workflows/BiOnymWF.java @@ -0,0 +1,80 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.workflows; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.GenericWorkflow; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherOutput; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.SingleEntry; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.FuzzyMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.GsayMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.LevensteinMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.matchers.TrigramMatcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.implementations.parsers.YasmeenParser; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces.Matcher; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class BiOnymWF extends GenericWorkflow{ + + public BiOnymWF (String sandboxFolder, int maxResults, HashMap globalparameters){ + super(sandboxFolder, globalparameters); + parser=new YasmeenParser(); + matchersList = new ArrayList(); + + matchersList.add(new GsayMatcher(sandboxFolder, 0.6, maxResults, globalparameters)); + matchersList.add(new FuzzyMatcher(sandboxFolder, 0.6, maxResults, globalparameters)); + matchersList.add(new LevensteinMatcher(sandboxFolder, 0.4, maxResults, globalparameters)); + matchersList.add(new TrigramMatcher(sandboxFolder, 0.4, maxResults, globalparameters)); + + postprocessor = null; + } + + @Override + protected MatcherOutput mergeOutputs(List outslist) { + MatcherOutput merged = new MatcherOutput(); + int k =0; + for (MatcherOutput out:outslist){ + int nelems = out.getEntriesNumber(); + for (int j=0;jStop"); + if (!merged.contains(se.originalName,se.targetScientificName, se.targetAuthor, se.targetID)){ + merged.addEntry(k, se.originalName, se.parsedScientificName, se.parsedAuthorship, + se.matchingScore, se.targetDataSource, se.targetID, + se.targetScientificName, se.targetAuthor, se.otherElements); + k++; + } + + } + + } + return merged; + } + + + public static void mainTest(String[] args) throws Exception{ + String sandboxFolder = "./PARALLEL_PROCESSING"; + HashMap globalparameters = new HashMap(); + globalparameters.put(YasmeenGlobalParameters.activatePreParsingProcessing, "true"); + globalparameters.put(YasmeenGlobalParameters.parserInputFileParam, "inputParser.txt"); + globalparameters.put(YasmeenGlobalParameters.parserOutputFileParam, "outputParser.txt"); + globalparameters.put(YasmeenGlobalParameters.parserNameParam, "SIMPLE"); + globalparameters.put(YasmeenGlobalParameters.taxaAuthorityFileParam, "ASFIS"); + globalparameters.put(YasmeenGlobalParameters.useStemmedGenusAndSpecies, "false"); + + BiOnymWF bionym = new BiOnymWF(sandboxFolder, 10, globalparameters); + + List rawEntries = new ArrayList(); + rawEntries.add("Salmo lucidus Richardson, 1836"); + rawEntries.add("Perca nilotica Linnaeus, 1758"); + + MatcherOutput output = bionym.executeChainedWorkflow(rawEntries); + int nEntries = output.getEntriesNumber(); + for (int i=0;i parameters); + + public MatcherOutput match(MatcherInput input) throws Exception; + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/interfaces/Parser.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/interfaces/Parser.java new file mode 100644 index 0000000..ef33f05 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/interfaces/Parser.java @@ -0,0 +1,13 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces; + +import java.util.HashMap; +import java.util.List; + +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherInput; + +public interface Parser { + + public void init (String sandboxfolder, HashMap parameters); + + public MatcherInput parse (List rawnames) throws Exception; +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/interfaces/PostProcessor.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/interfaces/PostProcessor.java new file mode 100644 index 0000000..38db243 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/interfaces/PostProcessor.java @@ -0,0 +1,17 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.interfaces; + +import java.util.HashMap; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.utils.Tuple; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherOutput; + +public interface PostProcessor { + + public void init(HashMap parameters); + + public MatcherOutput postprocessMatches(MatcherOutput rawnames); + + public HashMap produceAnalytics(); + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/test/TestBiOnymBiodiv.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/test/TestBiOnymBiodiv.java new file mode 100644 index 0000000..6b0e920 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/test/TestBiOnymBiodiv.java @@ -0,0 +1,66 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.test; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymBiodiv; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +import com.thoughtworks.xstream.XStream; + +public class TestBiOnymBiodiv { + + + public static void main(String[] args) throws Exception { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + String sandbox = "./PARALLEL_PROCESSING"; + String configfile = "testconfig.cfg"; + config.setPersistencePath(sandbox); + /* + config.setParam("DatabaseUserName", "gcube"); + config.setParam("DatabasePassword", "d4science2"); + config.setParam("DatabaseURL", "jdbc:postgresql://146.48.87.169/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); +*/ + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel"); +// config.setParam(BionymTransducer.originTableParam, "taxamatchinput1000"); +// config.setParam(BionymTransducer.rawnamesColumnParam, "rawstrings"); + //4 +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + //2 + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id471e6d50_d243_4112_bc07_e22152438e5c"); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + //FABIO DS: +// config.setParam(BionymTransducer.originTableParam, "generic_ide43477df_d9e6_4191_8a81_e94a0a2d16f8"); +// config.setParam(BionymTransducer.rawnamesColumnParam, "field0"); + + config.setParam(YasmeenGlobalParameters.parserNameParam,YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam,YasmeenGlobalParameters.BuiltinDataSources.OBIS.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing,"true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies,"false"); + config.setParam(YasmeenGlobalParameters.overallMaxResults,"10"); + + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + BufferedWriter oos = new BufferedWriter(new FileWriter(new File(sandbox, configfile))); + oos.write(new XStream().toXML(config)); + oos.close(); + + new BionymBiodiv().setup(config); + + new BionymBiodiv().executeNode(0, 1, 0, 2, false, sandbox, configfile, "test.log"); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/test/TestBiOnymGenericWF.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/test/TestBiOnymGenericWF.java new file mode 100644 index 0000000..e3353e2 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/test/TestBiOnymGenericWF.java @@ -0,0 +1,176 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.test; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +import com.thoughtworks.xstream.XStream; + +public class TestBiOnymGenericWF { + + public static void main1(String[] args) throws Exception { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + String sandbox = "./PARALLEL_PROCESSING"; + String configfile = "testconfig.cfg"; + config.setPersistencePath(sandbox); + /* + * config.setParam("DatabaseUserName", "gcube"); config.setParam("DatabasePassword", "d4science2"); config.setParam("DatabaseURL", "jdbc:postgresql://146.48.87.169/testdb"); config.setParam("DatabaseDriver", "org.postgresql.Driver"); + */ + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel"); + // 1000 + // config.setParam(BionymTransducer.originTableParam, "taxamatchinput1000"); + // config.setParam(BionymTransducer.rawnamesColumnParam, "rawstrings"); + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput"); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); + // 4 + // config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d"); + // config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + // FABIO DS: + // config.setParam(BionymTransducer.originTableParam, "generic_ide43477df_d9e6_4191_8a81_e94a0a2d16f8"); + // config.setParam(BionymTransducer.rawnamesColumnParam, "field0"); + + config.setParam(YasmeenGlobalParameters.parserNameParam, YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam, YasmeenGlobalParameters.BuiltinDataSources.FISHBASE.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing, "true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies, "false"); + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 1, YasmeenGlobalParameters.BuiltinMatchers.SOUNDEX.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 1, "0.2"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 1, "1"); + /* + * config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+2,YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+2,"0.2"); config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+2,"2"); + */ + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + BufferedWriter oos = new BufferedWriter(new FileWriter(new File(sandbox, configfile))); + oos.write(new XStream().toXML(config)); + oos.close(); + + new BionymFlexibleWorkflowTransducer().setup(config); + + new BionymFlexibleWorkflowTransducer().executeNode(0, 1, 0, 96, false, sandbox, configfile, "test.log"); + } + + public static void main2(String[] args) throws Exception { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + String sandbox = "./PARALLEL_PROCESSING"; + String configfile = "testconfig.cfg"; + config.setPersistencePath(sandbox); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "bionymfaked2red2"); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "sname"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "bionymfaked2test2"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "bionymfaked2test2"); + + config.setParam(YasmeenGlobalParameters.parserNameParam, YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam, YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing, "true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies, "false"); + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 1, YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 1, "0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 1, "10"); + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + BufferedWriter oos = new BufferedWriter(new FileWriter(new File(sandbox, configfile))); + oos.write(new XStream().toXML(config)); + oos.close(); + + new BionymFlexibleWorkflowTransducer().setup(config); + + new BionymFlexibleWorkflowTransducer().executeNode(0, 1, 0, 63, false, sandbox, configfile, "test.log"); + + } + + public static void main(String[] args) throws Exception { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + String sandbox = "./PARALLEL_PROCESSING"; + String configfile = "testconfig.cfg"; + config.setPersistencePath(sandbox); + config.setParam("DatabaseUserName", "utente"); + config.setParam("DatabasePassword", "d4science"); + config.setParam("DatabaseURL", "jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(YasmeenGlobalParameters.parserNameParam, YasmeenGlobalParameters.BuiltinParsers.NONE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam, YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing, "true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies, "false"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 1, YasmeenGlobalParameters.BuiltinMatchers.GSAy.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 1, "0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 1, "10"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 2, YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 2, "0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 2, "10"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 3, YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 3, "0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 3, "10"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix + "_" + 4, YasmeenGlobalParameters.BuiltinMatchers.TRIGRAM.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix + "_" + 4, "0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix + "_" + 4, "10"); + + AnalysisLogger.setLogger(config.getConfigPath() + AlgorithmConfiguration.defaultLoggerFile); + + config.setAgent("BIONYM"); + config.setPersistencePath("./"); + // config.setGcubeScope( "/gcube"); + config.setGcubeScope("/d4science.research-infrastructures.eu"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + // String directory = "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\BiOnym TaxaMatch\\BenchmarkTablesPrepared\\"; + String directory = "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\BiOnym TaxaMatch\\BenchmarkTablesPreparedReduced\\"; + File[] files = new File(directory).listFiles(); + int counter = 1; + for (File file : files) { + if (file.getName().endsWith(".prepr.csv")) { + + // if (file.getName().startsWith("real9.csv")){ + if (counter >= 5) { + + String tablename = "bionym" + file.getName().replace(".", ""); + System.out.println("Processing table " + tablename + " number:" + counter); + String outputtablename = "bionymoutsimple" + file.getName().replace(".", ""); + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, tablename); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "sname"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, outputtablename); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, outputtablename + "label"); + + BufferedWriter oos = new BufferedWriter(new FileWriter(new File(sandbox, configfile))); + oos.write(new XStream().toXML(config)); + oos.close(); + + new BionymFlexibleWorkflowTransducer().setup(config); + + new BionymFlexibleWorkflowTransducer().executeNode(0, 1, 0, 1024, false, sandbox, configfile, "test.log"); + + System.out.println("STOP FOR A WHILE " + counter + " of " + files.length); + Thread.sleep(1000); + break; + } + counter++; + } + + } + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/utils/YasmeenFileTools.java b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/utils/YasmeenFileTools.java new file mode 100644 index 0000000..c12fb8c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/nodes/transducers/bionym/utils/YasmeenFileTools.java @@ -0,0 +1,124 @@ +package org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.utils.Transformations; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherInput; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.MatcherOutput; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.abstracts.SingleEntry; + +public class YasmeenFileTools { + + public static MatcherInput getYasmeenParserOutput(String sandboxFolder, String filename) throws Exception{ + File inf = new File(sandboxFolder,filename); + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(inf), "UTF-8")); + + String line = br.readLine(); // skip headers + line = br.readLine(); + MatcherInput input = new MatcherInput(); + int i=0; + //PARSER;INPUT_DATA_SOURCE_ID;INPUT_DATA_ID;INPUT_DATA;PREPARSED_INPUT_DATA;PARSED_SCIENTIFIC_NAME;PARSED_AUTHORITY;POST_PARSED_SCIENTIFIC_NAME ;POST_PARSED_AUTHORITY + while (line!=null){ + List tokens = Transformations.parseCVSString(line, ";"); + if (tokens.size()>0){ + input.addEntry(i, tokens.get(3), tokens.get(7), tokens.get(8), null); + } + line = br.readLine(); + i++; + } + + br.close(); + return input; + + } + + public static MatcherOutput getYasmeenMatcherOutput(String sandboxFolder, String filename) throws Exception{ + File inf = new File(sandboxFolder,filename); + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(inf), "UTF-8")); + + String line = br.readLine(); // skip headers + line = br.readLine(); + + MatcherOutput output = new MatcherOutput(); + int i=0; +// SOURCE_DATASOURCE_ID SOURCE_ID SOURCE_DATA PRE_PARSED_SOURCE_DATA PARSED_SCIENTIFIC_NAME PARSED_AUTHORITY PARSER POST_PARSED_SCIENTIFIC_NAME POST_PARSED_AUTHORITY MATCHING_SCORE TARGET_DATA_SOURCE TARGET_DATA_ID TARGET_DATA_SCIENTIFIC_NAME TARGET_DATA_AUTHORITY TARGET_DATA_KINGDOM TARGET_DATA_PHYLUM TARGET_DATA_CLASS TARGET_DATA_ORDER TARGET_DATA_FAMILY TARGET_DATA_GENUS TARGET_DATA_SPECIES TARGET_DATA_VERNACULAR_NAMES + while (line!=null){ + AnalysisLogger.getLogger().debug("Yasmeen Output line:"+line); + List tokens = Transformations.parseCVSString(line, ";"); + if (tokens.size()>0){ + output.addEntry(i,tokens.get(2),tokens.get(7),tokens.get(8), + Double.parseDouble(tokens.get(9)),tokens.get(10),tokens.get(11),tokens.get(12),tokens.get(13),null); + } + line = br.readLine(); + i++; + } + + br.close(); + return output; + + } + + public static void writeYasmeenParserInput(String sandboxFolder, String filename, List rawNames) throws Exception{ + File inf = new File(sandboxFolder,filename); + if (inf.exists()) + inf.delete(); + + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(inf), "UTF-8")); + for (String rawName:rawNames) + bw.append(rawName+"\n"); + + bw.close(); + } + + @Deprecated + public static void writeYasmeenMatcherInput(String sandboxFolder, String filename, String parserName, MatcherInput input) throws Exception { + File inFile = new File(sandboxFolder,filename); + if (inFile.exists()) + inFile.delete(); + + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(inFile), "UTF-8")); + bw.write("PARSER;INPUT_DATA;PARSED_SCIENTIFIC_NAME;PARSED_AUTHORITY\n"); + int nInputs = input.getEntriesNumber(); + for (int i=0;i Config path "+config.getConfigPath()+" Persistence path: "+config.getPersistencePath()); + //default input and outputs + String defaultInputFileInTheScript = "tacsat.csv"; + String defaultOutputFileInTheScript = "tacsat_interpolated.csv"; + //input parameters: represent the context of the script. Values will be assigned in the R environment. + LinkedHashMap inputParameters = new LinkedHashMap(); + inputParameters.put("npoints",config.getParam("npoints")); + inputParameters.put("interval",config.getParam("interval")); + inputParameters.put("margin",config.getParam("margin")); + inputParameters.put("res",config.getParam("res")); + inputParameters.put("fm",config.getParam("fm")); + inputParameters.put("distscale",config.getParam("distscale")); + inputParameters.put("sigline",config.getParam("sigline")); + inputParameters.put("minspeedThr",config.getParam("minspeedThr")); + inputParameters.put("maxspeedThr",config.getParam("maxspeedThr")); + inputParameters.put("headingAdjustment",config.getParam("headingAdjustment")); + inputParameters.put("equalDist",config.getParam("equalDist").toUpperCase()); + //add static context variables + inputParameters.put("st", "c(minspeedThr,maxspeedThr)"); + inputParameters.put("fast", "TRUE"); + inputParameters.put("method", "\""+config.getParam("method")+"\""); + + AnalysisLogger.getLogger().debug("Starting SGVM Interpolation-> Input Parameters: "+inputParameters); + //if other code injection is required, put the strings to substitute as keys and the substituting ones as values + HashMap codeInjection = null; + //force the script to produce an output file, otherwise generate an exception + boolean scriptMustReturnAFile = true; + boolean uploadScriptOnTheInfrastructureWorkspace = false; //the Statistical Manager service will manage the upload + AnalysisLogger.getLogger().debug("SGVM Interpolation-> Executing the script "); + status = 10; + //execute the script in multi-user mode + scriptmanager.executeRScript(config, scriptName, inputFile, inputParameters, defaultInputFileInTheScript, defaultOutputFileInTheScript, codeInjection, scriptMustReturnAFile,uploadScriptOnTheInfrastructureWorkspace); + //assign the file path to an output variable for the SM + outputFile = scriptmanager.currentOutputFileName; + AnalysisLogger.getLogger().debug("SGVM Interpolation-> Output File is "+outputFile); + status = 100; + } + + @Override + protected void setInputParameters() { + //declare the input parameters the user will set: they will basically correspond to the R context + inputs.add(new PrimitiveType(File.class.getName(), null, PrimitiveTypes.FILE, "InputFile", "Input file in TACSAT format. E.g. http://goo.gl/i16kPw")); + addIntegerInput("npoints", "The number of pings or positions required between each real or actual vessel position or ping", "10"); + addIntegerInput("interval", "Average time in minutes between two adjacent datapoints", "120"); + addIntegerInput("margin", "Maximum deviation from specified interval to find adjacent datapoints (tolerance)", "10"); + addIntegerInput("res", "Number of points to use to create interpolation (including start and end point)", "100"); + addEnumerateInput(methodEnum.values(), "method","Set to cHs for cubic Hermite spline or SL for Straight Line interpolation", "cHs"); + addDoubleInput("fm", "The FM parameter in cubic interpolation", "0.5"); + addIntegerInput("distscale", "The DistScale parameter for cubic interpolation", "20"); + addDoubleInput("sigline", "The Sigline parameter in cubic interpolation", "0.2"); + addDoubleInput("minspeedThr", "A filter on the minimum speed to take into account for interpolation", "2"); + addDoubleInput("maxspeedThr", "A filter on the maximum speed to take into account for interpolation", "6"); + addIntegerInput("headingAdjustment", "Parameter to adjust the choice of heading depending on its own or previous point (0 or 1). Set 1 in case the heading at the endpoint does not represent the heading of the arriving vessel to that point but the departing vessel.", "0"); + inputs.add(new PrimitiveType(Boolean.class.getName(), null, PrimitiveTypes.BOOLEAN, "equalDist", "Whether the number of positions returned should be equally spaced or not", "true")); + } + + @Override + public StatisticalType getOutput() { + //return the output file by the procedure to the SM + PrimitiveType o = new PrimitiveType(File.class.getName(), new File(outputFile), PrimitiveTypes.FILE, "OutputFile", "Output file in TACSAT format."); + return o; + } + + @Override + public void shutdown() { + //in the case of forced shutdown, stop the R process + if (scriptmanager!=null) + scriptmanager.stop(); + System.gc(); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/scripts/OSCommand.java b/src/main/java/org/gcube/dataanalysis/executor/scripts/OSCommand.java new file mode 100644 index 0000000..deba476 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/scripts/OSCommand.java @@ -0,0 +1,101 @@ +package org.gcube.dataanalysis.executor.scripts; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.InputStreamReader; + +import org.gcube.common.core.utils.logging.GCUBELog; + +public class OSCommand { + + public static String ExecuteGetLine(String cmd, GCUBELog logger) { + + Process process = null; + String lastline = ""; + try { + if (logger == null) + System.out.println("ExecuteScript-> OSCommand-> Executing Control ->" + cmd); + else + logger.debug("ExecuteScript-> OSCommand-> Executing Control ->" + cmd); + + process = Runtime.getRuntime().exec(cmd); + + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = br.readLine(); + if (logger == null) + System.out.println("ExecuteScript-> OSCommand-> line->" + line); + else + logger.debug("ExecuteScript-> OSCommand-> line->" + line); + + while (line != null) { + try { + lastline = line; + if (logger == null) + System.out.println("ExecuteScript-> OSCommand-> line->" + line); + else + logger.debug("ExecuteScript-> OSCommand-> line->" + line); + line = br.readLine(); + } catch (EOFException e) { + if (logger == null) + System.out.println("ExecuteScript-> OSCommand -> Process Finished with EOF"); + else + logger.debug("ExecuteScript-> OSCommand -> Process Finished with EOF"); + break; + } catch (Exception e) { + line = "ERROR"; + break; + } + } + + if (logger == null) + System.out.println("ExecuteScript-> OSCommand -> Process Finished"); + else + logger.debug("ExecuteScript-> OSCommand -> Process Finished"); + } catch (Throwable e) { + if (logger == null) + System.out.println("ExecuteScript-> OSCommand-> error "); + else + logger.debug("ExecuteScript-> OSCommand-> error "); + e.printStackTrace(); + lastline = "ERROR"; + } + process.destroy(); + if (logger == null) + System.out.println("ExecuteScript-> OSCommand-> Process destroyed "); + else + logger.debug("ExecuteScript-> OSCommand-> Process destroyed "); + return lastline; + } + + public static boolean FileCopy(String origin, String destination) { + try { + + File inputFile = new File(origin); + System.out.println("ExecuteScript-> OSCommand-> FileCopy-> " + inputFile.length() + " to " + inputFile.canRead()); + int counterrors = 0; + while ((inputFile.length() == 0) && (counterrors < 10)) { + Thread.sleep(20); + counterrors++; + } + + File outputFile = new File(destination); + + FileReader in = new FileReader(inputFile); + FileWriter out = new FileWriter(outputFile); + int c; + + while ((c = in.read()) != -1) + out.write(c); + + in.close(); + out.close(); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/scripts/ScriptIOWorker.java b/src/main/java/org/gcube/dataanalysis/executor/scripts/ScriptIOWorker.java new file mode 100644 index 0000000..92017c7 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/scripts/ScriptIOWorker.java @@ -0,0 +1,19 @@ +package org.gcube.dataanalysis.executor.scripts; + +public class ScriptIOWorker { + + + public static String getString(String input){ + if (input!=null) + return input.replace("##", "."); + else + return input; + } + + public static String toInputString(String output){ + if (output!=null) + return output.replace(".", "##"); + else + return output; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/CustomRegressor.java b/src/main/java/org/gcube/dataanalysis/executor/tests/CustomRegressor.java new file mode 100644 index 0000000..0d3104b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/CustomRegressor.java @@ -0,0 +1,66 @@ +package org.gcube.dataanalysis.executor.tests; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; + +public class CustomRegressor { + + + public static void process(ComputationalAgent agent) throws Exception { + + if (agent != null) { + CustomRegressor tgs = new CustomRegressor(); + ThreadCalculator tc = tgs.new ThreadCalculator(agent); + Thread t = new Thread(tc); + t.start(); + while (agent.getStatus() < 100) { + + String resLoad = agent.getResourceLoad(); + String ress = agent.getResources(); + + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("STATUS: " + agent.getStatus()); + + Thread.sleep(10000); +// agent.shutdown(); + } + } else + System.out.println("Generator Algorithm Not Supported" ); + + System.out.println("-|"+agent.getStatus()); + } + + public class ThreadCalculator implements Runnable { + ComputationalAgent dg; + + public ThreadCalculator(ComputationalAgent dg) { + this.dg = dg; + } + + public void run() { + try { + + dg.compute(); + + } catch (Exception e) { + } + } + + } + + public static AlgorithmConfiguration getConfig() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName","gcube"); + config.setParam("DatabasePassword","d4science2"); + config.setParam("DatabaseURL","jdbc:postgresql://146.48.87.169/testdb"); + config.setParam("DatabaseDriver","org.postgresql.Driver"); + AnalysisLogger.setLogger(config.getConfigPath()+AlgorithmConfiguration.defaultLoggerFile); + return config; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym.java b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym.java new file mode 100644 index 0000000..b481db0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnym.java @@ -0,0 +1,113 @@ +package org.gcube.dataanalysis.executor.tests; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class RegressionTestBiOnym { + + public static void executeWF(String[] args) throws Exception { + // Generate + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(YasmeenGlobalParameters.parserNameParam,YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam,YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing,"true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies,"false"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+1,YasmeenGlobalParameters.BuiltinMatchers.GSAy.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+1,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+1,"10"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+2,YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+2,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+2,"10"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+3,YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+3,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+3,"10"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+4,YasmeenGlobalParameters.BuiltinMatchers.TRIGRAM.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+4,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+4,"10"); + + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel"); + //1000 +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput1000"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); + +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); + //4 + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d"); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + //2 +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id471e6d50_d243_4112_bc07_e22152438e5c"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + + //FABIO DS: +// config.setParam(BionymTransducer.originTableParam, "generic_ide43477df_d9e6_4191_8a81_e94a0a2d16f8"); +// config.setParam(BionymTransducer.rawnamesColumnParam, "field0"); + + config.setAgent("BIONYM"); + + config.setPersistencePath("./"); + config.setGcubeScope( "/gcube"); +// config.setGcubeScope( "/d4science.research-infrastructures.eu"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + + generate(config); + + } + + public static void main(String []args) throws Exception{ + + for (int i=0;i<1;i++){ + executeWF(args); + } + } + + + public static void generate(AlgorithmConfiguration config) throws Exception { + + D4ScienceDistributedProcessing generator = new D4ScienceDistributedProcessing(config); + generator.init(); + + if (generator != null) { + long t0 = System.currentTimeMillis(); + TestGenerator tgs = new TestGenerator(generator); + Thread t = new Thread(tgs); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(2000); + } + System.out.println("FINAL STATUS: " + generator.getStatus()+ " ELAPSED "+(System.currentTimeMillis()-t0)); + + } + else + System.out.println("Generator Algorithm Not Supported"); + +// generator.generate(); +// } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnymProduzione.java b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnymProduzione.java new file mode 100644 index 0000000..a5095cd --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestBiOnymProduzione.java @@ -0,0 +1,100 @@ +package org.gcube.dataanalysis.executor.tests; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class RegressionTestBiOnymProduzione { + + public static void executeWF(String[] args) throws Exception { + // Generate + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(YasmeenGlobalParameters.parserNameParam,YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam,YasmeenGlobalParameters.BuiltinDataSources.FISHBASE.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing,"true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies,"false"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+1,YasmeenGlobalParameters.BuiltinMatchers.GSAy.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+1,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+1,"10"); + + + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, "taxamatchoutputlocal"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, "taxamatchoutputlabel"); + //1000 + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput1000"); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); + +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "taxamatchinput"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "rawstrings"); + //4 +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id1ecb405c_980f_47a4_926a_3043d065fc7d"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + //2 +// config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, "generic_id471e6d50_d243_4112_bc07_e22152438e5c"); +// config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "field0"); + + //FABIO DS: +// config.setParam(BionymTransducer.originTableParam, "generic_ide43477df_d9e6_4191_8a81_e94a0a2d16f8"); +// config.setParam(BionymTransducer.rawnamesColumnParam, "field0"); + + config.setAgent("BIONYM"); + + config.setPersistencePath("./"); + config.setGcubeScope( "/gcube"); +// config.setGcubeScope( "/d4science.research-infrastructures.eu"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + + generate(config); + + } + + public static void main(String []args) throws Exception{ + + for (int i=0;i<1;i++){ + executeWF(args); + } + } + + + public static void generate(AlgorithmConfiguration config) throws Exception { + + D4ScienceDistributedProcessing generator = new D4ScienceDistributedProcessing(config); + generator.init(); + + if (generator != null) { + long t0 = System.currentTimeMillis(); + TestGenerator tgs = new TestGenerator(generator); + Thread t = new Thread(tgs); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(2000); + } + System.out.println("FINAL STATUS: " + generator.getStatus()+ " ELAPSED "+(System.currentTimeMillis()-t0)); + + } + else + System.out.println("Generator Algorithm Not Supported"); + +// generator.generate(); +// } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestCMSY.java b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestCMSY.java new file mode 100644 index 0000000..3fa4a3c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestCMSY.java @@ -0,0 +1,65 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; + +public class RegressionTestCMSY { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + +public static AlgorithmConfiguration getConfig() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://dbtest.research-infrastructures.eu/testdb"); + config.setParam("DatabaseDriver","org.postgresql.Driver"); + AnalysisLogger.setLogger(config.getConfigPath()+AlgorithmConfiguration.defaultLoggerFile); + return config; + } + + public static void main(String[] args) throws Exception { + + System.out.println("TEST 1"); + + List generators = GeneratorsFactory.getGenerators(testCMSY()); + generators.get(0).init(); + CustomRegressor.process(generators.get(0)); + generators = null; + + } + + private static AlgorithmConfiguration testCMSY() { + + AlgorithmConfiguration config = getConfig(); + config.setNumberOfResources(5); + config.setModel("CMSY"); + + config.setParam("UserName", "gianpaolo.coro"); + config.setGcubeScope("/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + + + config.setParam("IDsFile","https://dl.dropboxusercontent.com/u/12809149/WKLIFE4ID.csv"); + config.setParam("StocksFile","https://dl.dropboxusercontent.com/u/12809149/WKLIFE4Stocks.csv"); + + //config.setParam("IDsFile","http://goo.gl/9rg3qK"); + // config.setParam("StocksFile","http://goo.gl/Mp2ZLY"); +// config.setParam("StocksFile","http://goo.gl/btuIIe"); +// config.setParam("SelectedStock","Pan_bor_1"); + config.setParam("SelectedStock","HLH_M08"); + + return config; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestGenerators.java b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestGenerators.java new file mode 100644 index 0000000..f540319 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestGenerators.java @@ -0,0 +1,107 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; + +public class RegressionTestGenerators { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + + public static void main(String[] args) throws Exception { + + System.out.println("TEST 1"); + + List generators = GeneratorsFactory.getGenerators(testAquamapsSuitable2050()); + generators.get(0).init(); + Regressor.process(generators.get(0)); + generators = null; + + } + + private static AlgorithmConfiguration testAquamapsSuitable() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setNumberOfResources(5); + config.setModel("AQUAMAPS_SUITABLE"); + + config.setParam("UserName", "gianpaolo.coro"); + config.setGcubeScope("/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DistributionTable", "hspec_suitable_test_parallel"); + config.setParam("CsquarecodesTable", "hcaf_d"); + config.setParam("EnvelopeTable", "hspen_micro_1"); + config.setParam("OccurrencePointsTable", "occurrencecells"); + config.setParam("CreateTable", "true"); + + return config; + } + + private static AlgorithmConfiguration testAquamapsNative() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setNumberOfResources(5); + config.setModel("AQUAMAPS_NATIVE"); + + config.setParam("UserName", "gianpaolo.coro"); + config.setGcubeScope("/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DistributionTable", "hspec_native_test_parallel2_1000"); + config.setParam("CsquarecodesTable", "hcaf_d"); +// config.setParam("EnvelopeTable", "hspen_micro_1"); + config.setParam("EnvelopeTable", "hspen_mini_1000"); + config.setParam("OccurrencePointsTable", "occurrencecells"); + config.setParam("CreateTable", "true"); + + return config; + } + + + private static AlgorithmConfiguration testAquamapsNative2050() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setNumberOfResources(5); + config.setModel("AQUAMAPS_NATIVE_2050"); + + config.setParam("UserName", "gianpaolo.coro"); + config.setGcubeScope("/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DistributionTable", "hspec_native2050_test_parallel"); + config.setParam("CsquarecodesTable", "hcaf_d"); +// config.setParam("EnvelopeTable", "hspen_micro_1"); + config.setParam("EnvelopeTable", "hspen_validation"); + config.setParam("OccurrencePointsTable", "occurrencecells"); + config.setParam("CreateTable", "true"); + + return config; + } + + private static AlgorithmConfiguration testAquamapsSuitable2050() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setNumberOfResources(5); + config.setModel("AQUAMAPS_SUITABLE_2050"); + + config.setParam("UserName", "gianpaolo.coro"); + config.setGcubeScope("/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DistributionTable", "hspec_suitable2050_test_parallel_validation2"); + config.setParam("CsquarecodesTable", "hcaf_d"); + config.setParam("EnvelopeTable", "hspen_micro_1"); +// config.setParam("EnvelopeTable", "hspen_validation"); + config.setParam("OccurrencePointsTable", "occurrencecells"); + config.setParam("CreateTable", "true"); + + return config; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestLWR.java b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestLWR.java new file mode 100644 index 0000000..9310810 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestLWR.java @@ -0,0 +1,63 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; + +public class RegressionTestLWR { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + +public static AlgorithmConfiguration getConfig() { + + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://dbtest.research-infrastructures.eu/testdb"); + config.setParam("DatabaseDriver","org.postgresql.Driver"); + AnalysisLogger.setLogger(config.getConfigPath()+AlgorithmConfiguration.defaultLoggerFile); + return config; + } + + public static void main(String[] args) throws Exception { + + System.out.println("TEST 1"); + + List generators = GeneratorsFactory.getGenerators(testLWR()); + generators.get(0).init(); + CustomRegressor.process(generators.get(0)); + generators = null; + + } + + private static AlgorithmConfiguration testLWR() { + + AlgorithmConfiguration config = getConfig(); + config.setNumberOfResources(5); + config.setModel("LWR"); + + config.setParam("UserName", "gianpaolo.coro"); + config.setGcubeScope("/gcube"); +// config.setGcubeScope("/d4science.research-infrastructures.eu"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + //config.setParam("LWR_Input", "generic_iddd4bbfc2_12bd_4132_a4fc_2d64078e90e9"); + config.setParam("LWR_Input", "generic_id4834c9fb_9e57_4bef_b0b0_0f9b000ce2bb"); + + config.setParam("TableLabel", "lwrout"); + + config.setParam("FamilyColumn", "\"family\""); + config.setParam("RealOutputTable", "lwr6"); + + return config; + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestTransducers.java b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestTransducers.java new file mode 100644 index 0000000..60ef3b0 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/RegressionTestTransducers.java @@ -0,0 +1,161 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.evaluation.bioclimate.InterpolateTables.INTERPOLATIONFUNCTIONS; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.interfaces.Transducerer; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; + +public class RegressionTestTransducers { + +public static void main(String[] args) throws Exception { + + System.out.println("TEST 1"); + List trans = null; + /* + trans = TransducerersFactory.getTransducerers(testConfigLocal()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; + + trans = TransducerersFactory.getTransducerers(testConfigLocal2()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; + + trans = TransducerersFactory.getTransducerers(testConfigLocal3()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; + + trans = TransducerersFactory.getTransducerers(testConfigLocal4()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; + + trans = TransducerersFactory.getTransducerers(testConfigLocal5()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; + + trans = TransducerersFactory.getTransducerers(testConfigLocal6()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; + */ + trans = TransducerersFactory.getTransducerers(testConfigLocal5()); + trans.get(0).init(); + Regressor.process(trans.get(0)); + trans = null; +} + + + private static AlgorithmConfiguration testConfigLocal() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("BIOCLIMATE_HSPEC"); + config.setParam("HSPEC_TABLE_LIST", "hspec_validation"+AlgorithmConfiguration.getListSeparator()+"hspec_validation2"); + config.setParam("HSPEC_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"); + config.setParam("Threshold", "0.5"); + + return config; + } + + private static AlgorithmConfiguration testConfigLocal2() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("BIOCLIMATE_HCAF"); + config.setParam("HCAF_TABLE_LIST","hcaf_d"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01332632269756"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2016_linear_01336062995861"+AlgorithmConfiguration.getListSeparator()+"hcaf_d_2050"); + config.setParam("HCAF_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"); + + return config; + } + + private static AlgorithmConfiguration testConfigLocal3() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("BIOCLIMATE_HSPEN"); + config.setParam("HSPEN_TABLE_LIST","hspen"+AlgorithmConfiguration.getListSeparator()+"hspen_2016"+AlgorithmConfiguration.getListSeparator()+"hspen_2020"+AlgorithmConfiguration.getListSeparator()+"hspen_2050"); + config.setParam("HSPEN_TABLE_NAMES", "test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"+AlgorithmConfiguration.getListSeparator()+"test"); + + return config; + } + + + private static AlgorithmConfiguration testConfigLocal4() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("HCAF_INTERPOLATION"); + + config.setParam("FirstHCAF","hcaf_d"); + config.setParam("SecondHCAF","hcaf_d_2050"); + config.setParam("YearStart","2012"); + config.setParam("YearEnd","2050"); + config.setParam("NumberOfInterpolations","2"); + config.setParam("InterpolationFunction",INTERPOLATIONFUNCTIONS.LINEAR.name()); + + return config; + } + + private static AlgorithmConfiguration testConfigLocal5() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("OCCURRENCES_MERGER"); + + config.setParam("longitudeColumn", "decimallongitude"); + config.setParam("latitudeColumn", "decimallatitude"); + config.setParam("recordedByColumn", "recordedby"); + config.setParam("scientificNameColumn", "scientificname"); + config.setParam("eventDateColumn", "eventdate"); + config.setParam("lastModificationColumn", "modified"); + config.setParam("rightTableName", "speciesset2"); + config.setParam("leftTableName", "speciesset1"); + config.setParam("finalTableName", "testspeciesmerged"); + config.setParam("spatialTolerance", "0.5"); + config.setParam("confidence", "80"); + + config.setGcubeScope("/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + return config; + } + + private static AlgorithmConfiguration testConfigLocal6() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("OCCURRENCES_INSEAS_ONEARTH"); + + config.setParam("longitudeColumn", "decimallongitude"); + config.setParam("latitudeColumn", "decimallatitude"); + config.setParam("OccurrencePointsTableName", "whitesharkoccurrences2"); + config.setParam("finalTableName", "whitesharkoccurrencesfilteredseas"); + config.setParam("FilterType", "IN_THE_WATER"); +// config.setParam("FilterType", "ON_EARTH"); + + return config; + } + + private static AlgorithmConfiguration testConfigLocal7() { + + AlgorithmConfiguration config = Regressor.getConfig(); + config.setAgent("OCCURRENCES_DUPLICATE_DELETER"); + + config.setParam("longitudeColumn", "decimallongitude"); + config.setParam("latitudeColumn", "decimallatitude"); + config.setParam("recordedByColumn", "recordedby"); + config.setParam("scientificNameColumn", "scientificname"); + config.setParam("eventDateColumn", "eventdate"); + config.setParam("lastModificationColumn", "modified"); + config.setParam("OccurrencePointsTableName", "whitesharkoccurrences2"); + config.setParam("finalTableName", "whitesharkoccurrencesnoduplicates"); + config.setParam("spatialTolerance", "0.5"); + config.setParam("confidence", "80"); + + return config; + } + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnym1_0_0.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnym1_0_0.java new file mode 100644 index 0000000..c992be5 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnym1_0_0.java @@ -0,0 +1,79 @@ +package org.gcube.dataanalysis.executor.tests; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymWorkflow; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.CometMatcherManager; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.EVBPreprocessing; + +public class TestBiOnym1_0_0 { + + public static void main(String[] args) throws Exception { + // Generate + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + + config.setParam(BionymWorkflow.destinationTableParam, "taxamatchoutput"); + config.setParam(BionymWorkflow.destinationTableLable, "taxamatchoutputlabel"); + config.setParam(BionymWorkflow.originTableParam, "taxamatchinput"); + config.setParam(BionymWorkflow.rawnamesColumnParam, "rawstrings"); + config.setParam(BionymWorkflow.parserParam, CometMatcherManager.Parsers.SIMPLE.name()); + config.setParam(BionymWorkflow.referenceParam, CometMatcherManager.Reference.ASFIS.name()); + config.setParam(BionymWorkflow.soundexweightParam, CometMatcherManager.Weights.EDIT_DISTANCE.name()); + config.setParam(BionymWorkflow.doPreprocessParam , EVBPreprocessing.Preprocessors.EXPERT_RULES.name()); + config.setParam(BionymWorkflow.maxMatchesParam , "10"); + config.setAgent("BIONYM"); + + config.setPersistencePath("./"); + config.setGcubeScope( "/gcube"); +// config.setGcubeScope( "/d4science.research-infrastructures.eu"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); +// config.setParam("DatabaseURL","jdbc:postgresql://dbtest.research-infrastructures.eu/testdb"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + +// config.setParam("DatabaseUserName", "gcube"); +// config.setParam("DatabasePassword", "d4science2"); +// config.setParam("DatabaseURL", "jdbc:postgresql://146.48.87.169/testdb"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + + generate(config); + + } + + + + + public static void generate(AlgorithmConfiguration config) throws Exception { + + D4ScienceDistributedProcessing generator = new D4ScienceDistributedProcessing(config); + generator.init(); + + if (generator != null) { + long t0 = System.currentTimeMillis(); + TestGenerator tgs = new TestGenerator(generator); + Thread t = new Thread(tgs); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(2000); + } + System.out.println("FINAL STATUS: " + generator.getStatus()+ " ELAPSED "+(System.currentTimeMillis()-t0)); + + } + else + System.out.println("Generator Algorithm Not Supported"); + +// generator.generate(); +// } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymEvaluation.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymEvaluation.java new file mode 100644 index 0000000..bf8a3f1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymEvaluation.java @@ -0,0 +1,124 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.io.File; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class TestBiOnymEvaluation { + + public static void main(String[] args) throws Exception { + // Generate + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(YasmeenGlobalParameters.parserNameParam,YasmeenGlobalParameters.BuiltinParsers.GNI.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam,YasmeenGlobalParameters.BuiltinDataSources.WORMS_PISCES.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing,"true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies,"false"); + + /* + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+1,YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+1,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+1,"10"); + */ + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+1,YasmeenGlobalParameters.BuiltinMatchers.GSAy.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+1,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+1,"10"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+2,YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+2,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+2,"10"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+3,YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+3,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+3,"10"); + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+4,YasmeenGlobalParameters.BuiltinMatchers.TRIGRAM.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+4,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+4,"10"); + +// System.exit(0); + config.setAgent("BIONYM"); + config.setPersistencePath("./"); + config.setGcubeScope( "/gcube"); +// config.setGcubeScope( "/d4science.research-infrastructures.eu"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); +// String directory = "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\BiOnym TaxaMatch\\BenchmarkTablesPrepared\\"; +// String directory = "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\BiOnym TaxaMatch\\BenchmarkTablesPreparedReduced\\"; + String directory = "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\BiOnym TaxaMatch\\BenchmarkSimulatedReduced\\"; +// String directory = "C:\\Users\\coro\\Desktop\\DATABASE e NOTE\\Experiments\\BiOnym TaxaMatch\\BenchmarkWard\\"; + File[] files = new File(directory).listFiles(); + int counter = 1; + long t0=System.currentTimeMillis(); + for (File file:files){ + if (file.getName().endsWith(".prepr.csv")){ + +// if (file.getName().startsWith("real9.csv")){ + + if (counter>=0){ +// String tablename = "bionymreal5csv6"; + String tablename = "bionym" + file.getName().replace(".", ""); + System.out.println("Processing table "+tablename+" number:"+counter); + String outputtablename = "bionymoutsimulgni2"+file.getName().replace(".",""); + config.setParam(BionymFlexibleWorkflowTransducer.originTableParam, tablename); + config.setParam(BionymFlexibleWorkflowTransducer.rawnamesColumnParam, "sname"); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableParam, outputtablename); + config.setParam(BionymFlexibleWorkflowTransducer.destinationTableLableParam, outputtablename+"label"); + generate(config); + + System.out.println("STOP FOR A WHILE "+counter); + Thread.sleep(1000); +// break; + } + counter++; + + } + + } + + System.out.println("OVERALL COMPUTATION ON ALL TABLES: "+(System.currentTimeMillis()-t0)+" ms"); + } + + + + + public static void generate(AlgorithmConfiguration config) throws Exception { + + D4ScienceDistributedProcessing generator = new D4ScienceDistributedProcessing(config); + generator.init(); + + if (generator != null) { + long t0 = System.currentTimeMillis(); + TestGenerator tgs = new TestGenerator(generator); + Thread t = new Thread(tgs); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(5000); + } + System.out.println("FINAL STATUS: " + generator.getStatus()+ " ELAPSED "+(System.currentTimeMillis()-t0)); + + } + else + System.out.println("Generator Algorithm Not Supported"); + +// generator.generate(); +// } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocal.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocal.java new file mode 100644 index 0000000..4fdb614 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocal.java @@ -0,0 +1,73 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.io.File; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class TestBiOnymLocal { + + public static void main(String[] args) throws Exception { + // Generate + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(YasmeenGlobalParameters.parserNameParam,YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam,YasmeenGlobalParameters.BuiltinDataSources.FISHBASE.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing,"true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies,"false"); + + config.setParam(YasmeenGlobalParameters.performanceParam,YasmeenGlobalParameters.Performance.HIGH_SPEED.name()); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+3,YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+3,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+3,"10"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+1,YasmeenGlobalParameters.BuiltinMatchers.GSAy.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+1,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+1,"10"); + + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+2,YasmeenGlobalParameters.BuiltinMatchers.FUZZYMATCH.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+2,"0.6"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+2,"10"); + + /* + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+3,YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+3,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+3,"10"); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+4,YasmeenGlobalParameters.BuiltinMatchers.TRIGRAM.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+4,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+4,"10"); + */ + config.setAgent("BIONYM_LOCAL"); + config.setPersistencePath("./"); + config.setGcubeScope( "/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + config.setParam(BionymLocalTransducer.speciesName, "Gadus morhua (Linnaeus, 1758)"); + List transducers = TransducerersFactory.getTransducerers(config); + ComputationalAgent transducer =transducers.get(0); + transducer.init(); + + List types = transducer.getInputParameters(); +// System.out.println(types); + Regressor.process(transducer); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocalFastVersion.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocalFastVersion.java new file mode 100644 index 0000000..da2141a --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestBiOnymLocalFastVersion.java @@ -0,0 +1,54 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.io.File; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymFlexibleWorkflowTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.BionymLocalTransducer; +import org.gcube.dataanalysis.executor.nodes.transducers.bionym.utils.YasmeenGlobalParameters; + +public class TestBiOnymLocalFastVersion { + + public static void main(String[] args) throws Exception { + // Generate + AlgorithmConfiguration config = new AlgorithmConfiguration(); + + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://statistical-manager.d.d4science.research-infrastructures.eu/testdb"); + + config.setParam(YasmeenGlobalParameters.parserNameParam,YasmeenGlobalParameters.BuiltinParsers.SIMPLE.name()); + config.setParam(YasmeenGlobalParameters.taxaAuthorityFileParam,YasmeenGlobalParameters.BuiltinDataSources.COL_FULL.name()); + config.setParam(YasmeenGlobalParameters.activatePreParsingProcessing,"true"); + config.setParam(YasmeenGlobalParameters.useStemmedGenusAndSpecies,"false"); + + config.setParam(YasmeenGlobalParameters.performanceParam,YasmeenGlobalParameters.Performance.MAX_ACCURACY.name()); + + config.setParam(BionymFlexibleWorkflowTransducer.matcherParamPrefix+"_"+3,YasmeenGlobalParameters.BuiltinMatchers.LEVENSHTEIN.name()); + config.setParam(BionymFlexibleWorkflowTransducer.thresholdParamPrefix+"_"+3,"0.4"); + config.setParam(BionymFlexibleWorkflowTransducer.maxresultsParamPrefix+"_"+3,"10"); + + config.setAgent("BIONYM_LOCAL"); + config.setPersistencePath("./"); + config.setConfigPath("./cfg/"); + config.setGcubeScope( "/gcube"); + config.setParam("ServiceUserName", "gianpaolo.coro"); + config.setParam("DatabaseDriver", "org.postgresql.Driver"); + config.setParam(BionymLocalTransducer.speciesName, "Gadus morhua (Linnaeus, 1758)"); + List transducers = TransducerersFactory.getTransducerers(config); + ComputationalAgent transducer =transducers.get(0); + transducer.init(); + + List types = transducer.getInputParameters(); + + Regressor.process(transducer); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceMaps.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceMaps.java new file mode 100644 index 0000000..71e4e4d --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceMaps.java @@ -0,0 +1,93 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.user.GeneratorT; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; + +public class TestD4ScienceMaps { + + public static void main(String[] args) throws Exception { + String configPath = "./cfg/"; + String csquareTable = "hcaf_d"; + String preprocessedTable = "maxminlat_hspen"; + String envelopeTable = "hspen_mini_10"; + int numberOfResources = 1; + String userName = "gianpaolo.coro"; + String generatorName = "AQUAMAPS_SUITABLE"; + String scope = "/gcube"; + String finalDistributionTable = "hspec_suitable_executor6"; +// Train +// ModelerT.train(ModelerT.getTrainingConfig(modelName, absenceTable, presenceTable, speciesCode, userName, neuralNetworkLayers, configPath)); + + // Generate + AlgorithmConfiguration config = GeneratorT.getGenerationConfig(numberOfResources, generatorName, envelopeTable, preprocessedTable, "", userName, csquareTable, finalDistributionTable, configPath); + config.setPersistencePath("./"); + config.setGcubeScope(scope); + config.setParam("ServiceUserName", "gianpaolo.coro"); + config.setParam("DatabaseUserName","utente"); + config.setParam("DatabasePassword","d4science"); + config.setParam("DatabaseURL","jdbc:postgresql://dbtest.research-infrastructures.eu/aquamapsorgupdated"); + config.setParam("DatabaseDriver","org.hibernate.dialect.PostgreSQLDialect"); + + List endpoints = new ArrayList(); + + endpoints.add("http://node11.d.d4science.research-infrastructures.eu:9000/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node26.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node29.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node28.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + + endpoints.add("http://node30.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node31.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node32.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node34.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + endpoints.add("http://node33.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + + endpoints.add("http://node27.d.d4science.research-infrastructures.eu:8080/wsrf/services/gcube/vremanagement/executor/engine"); + +// config.setEndpoints(endpoints); + generate(config); + /* + finalDistributionTable = "hspec_suitable_executor_2"; + config.setParam("DistributionTable", finalDistributionTable); + generate(config); + */ + + } + + + + + public static void generate(AlgorithmConfiguration config) throws Exception { + + D4ScienceDistributedProcessing generator = new D4ScienceDistributedProcessing(config); + generator.init(); + + if (generator != null) { + long t0 = System.currentTimeMillis(); + TestGenerator tgs = new TestGenerator(generator); + Thread t = new Thread(tgs); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); + String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); + System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(2000); + } + System.out.println("FINAL STATUS: " + generator.getStatus()+ " ELAPSED "+(System.currentTimeMillis()-t0)); + + } + else + System.out.println("Generator Algorithm Not Supported"); + +// generator.generate(); +// } + } +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps.java new file mode 100644 index 0000000..501ceb4 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestD4ScienceQueueMaps.java @@ -0,0 +1,91 @@ +package org.gcube.dataanalysis.executor.tests; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.user.GeneratorT; +import org.gcube.dataanalysis.executor.generators.D4ScienceDistributedProcessing; + +public class TestD4ScienceQueueMaps { + + public static void main(String[] args) throws Exception { + int k1=0; + int k2=3; + for (int i=k1;i generators = GeneratorsFactory.getGenerators(config); + ComputationalAgent generator = generators.get(0); + + if (generator != null) { + generator.init(); + TestGenerator tgs = new TestGenerator(generator); + Thread t = new Thread(tgs); + t.start(); + while (generator.getStatus() < 100) { + + String resLoad = generator.getResourceLoad(); + String ress = generator.getResources(); +// String species = generator.getLoad(); + System.out.println("LOAD: " + resLoad); + System.out.println("RESOURCES: " + ress); +// System.out.println("SPECIES: " + species); + System.out.println("STATUS: " + generator.getStatus()); + Thread.sleep(1000); + } + } + else + System.out.println("Generator Algorithm Not Supported"); + } + + + public static AlgorithmConfiguration getGenerationConfig(int numberOfResources,String algorithmName,String envelopeTable,String preprocessedTable,String speciesName,String userName,String csquareTable,String finalDistributionTable,String configPath) { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath(configPath); + config.setPersistencePath(configPath); + config.setNumberOfResources(numberOfResources); + config.setModel(algorithmName); + + config.setParam("EnvelopeTable", envelopeTable); + config.setParam("PreprocessedTable", preprocessedTable); + + config.setParam("SpeciesName", speciesName); + config.setParam("CsquarecodesTable", csquareTable); + config.setParam("DistributionTable", finalDistributionTable); + config.setParam("CreateTable", "true"); + config.setParam("UserName",userName); + + return config; + } + + + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestGetRunningExecutor.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestGetRunningExecutor.java new file mode 100644 index 0000000..949e709 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestGetRunningExecutor.java @@ -0,0 +1,73 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.axis.message.addressing.EndpointReferenceType; +import org.gcube.common.core.contexts.GHNContext; +import org.gcube.common.core.informationsystem.client.AtomicCondition; +import org.gcube.common.core.informationsystem.client.ISClient; +import org.gcube.common.core.informationsystem.client.RPDocument; +import org.gcube.common.core.informationsystem.client.queries.WSResourceQuery; +import org.gcube.common.core.scope.GCUBEScope; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; + +public class TestGetRunningExecutor { + + + private static int findNodes(String scopeString) throws Exception { + AnalysisLogger.getLogger().debug("*****"); + GCUBEScope scope = GCUBEScope.getScope(scopeString); + ISClient client = GHNContext.getImplementation(ISClient.class); + WSResourceQuery wsquery = client.getQuery(WSResourceQuery.class); + wsquery.addAtomicConditions(new AtomicCondition("//gc:ServiceName", "Executor")); +// wsquery.addAtomicConditions(new AtomicCondition("/child::*[local-name()='Task']/name", "ExecutorScript")); + wsquery.addAtomicConditions(new AtomicCondition("/child::*[local-name()='Task']/name[text()='ExecutorScript']", "ExecutorScript")); + List listdoc = client.execute(wsquery, scope); + EndpointReferenceType epr = null; + ArrayList eprs = new ArrayList(); + int numberOfEP = 0; + for (RPDocument resource : listdoc) { + epr = resource.getEndpoint(); + numberOfEP++; + AnalysisLogger.getLogger().debug("*** " + epr); + eprs.add(epr); + } + AnalysisLogger.getLogger().debug("Found " + numberOfEP + " endpoints"); + AnalysisLogger.getLogger().debug("-> "+ eprs); + + return numberOfEP; + } + + + private static int findRunningNodes(String scopeString) throws Exception { + AnalysisLogger.getLogger().debug("*****"); + GCUBEScope scope = GCUBEScope.getScope(scopeString); + System.out.println("BROKER:"+scope.getServiceMap().getEndpoints(GHNContext.MSGBROKER).iterator().next().getAddress().toString()); + ISClient client = GHNContext.getImplementation(ISClient.class); + WSResourceQuery wsquery = client.getQuery(WSResourceQuery.class); + wsquery.addAtomicConditions(new AtomicCondition("//gc:ServiceName", "Executor")); + wsquery.addAtomicConditions(new AtomicCondition(" /child::*[local-name()='State']", "RUNNING")); + List listdoc = client.execute(wsquery, scope); + EndpointReferenceType epr = null; + ArrayList eprs = new ArrayList(); + int numberOfEP = 0; + for (RPDocument resource : listdoc) { + epr = resource.getEndpoint(); + numberOfEP++; + AnalysisLogger.getLogger().debug("*** " + epr); + eprs.add(epr); + } + AnalysisLogger.getLogger().debug("Found " + numberOfEP + " endpoints"); + AnalysisLogger.getLogger().debug("-> "+ eprs); + return numberOfEP; + } + + public static void main(String[] args) throws Exception{ + String scope = "/gcube"; + AnalysisLogger.setLogger("./cfg/ALog.properties"); + findNodes(scope); +// findRunningNodes(scope); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestRemoteJobLaunch.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestRemoteJobLaunch.java new file mode 100644 index 0000000..1bd0ed6 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestRemoteJobLaunch.java @@ -0,0 +1,29 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.executor.job.management.RemoteJobManager; + +public class TestRemoteJobLaunch { + + + public static void main(String [] args) throws Exception{ + String scope = "/gcube"; + String serviceClass = "TestGP"; + String serviceName = "TestGPHome"; + String owner = "GP"; + String directory = "./shipping/"; + String remotedirectory = "/shipping/"; + String tempDir = "./"; + String scriptName = "execute.sh"; + int numberOfNodes = 1; + List argums = new ArrayList(); + argums.add("0_178204_0_3_./"); + AnalysisLogger.setLogger("./cfg/ALog.properties"); + RemoteJobManager job = new RemoteJobManager(scope,numberOfNodes); +// job.uploadAndExecute(serviceClass, serviceName, owner, directory, remotedirectory, tempDir, scriptName, argums); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestSGVMInterpolation.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestSGVMInterpolation.java new file mode 100644 index 0000000..09de19d --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestSGVMInterpolation.java @@ -0,0 +1,55 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.List; + +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.PrimitiveType; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; +import org.gcube.dataanalysis.ecoengine.test.regression.Regressor; + +public class TestSGVMInterpolation { + + public static void main(String[] args) throws Exception { + // setup the configuration + AlgorithmConfiguration config = new AlgorithmConfiguration(); + // set the path to the cfg folder and to the PARALLEL_PROCESSING folder + config.setConfigPath("./cfg/"); + config.setPersistencePath("./PARALLEL_PROCESSING"); + //set the user's inputs. They will passed by the SM to the script in the following way: +// config.setParam("InputFile", "/tacsatmini.csv"); //put the absolute path to the input file + config.setParam("InputFile", "./tacsatmini.csv"); //put the absolute path to the input file + config.setParam("npoints", "10"); + config.setParam("interval", "120"); + config.setParam("margin", "10"); + config.setParam("res", "100"); + config.setParam("method", "SL"); + config.setParam("fm", "0.5"); + config.setParam("distscale", "20"); + config.setParam("sigline", "0.2"); + config.setParam("minspeedThr", "2"); + config.setParam("maxspeedThr", "6"); + config.setParam("headingAdjustment", "0"); + config.setParam("equalDist", "true"); + + //set the scope and the user (optional for this test) + config.setGcubeScope( "/gcube/devsec/devVRE"); + config.setParam("ServiceUserName", "test.user"); + + //set the name of the algorithm to call, as is is in the transducerer.properties file + config.setAgent("SGVM_INTERPOLATION"); + + //recall the transducerer with the above name + List transducers = TransducerersFactory.getTransducerers(config); + ComputationalAgent transducer =transducers.get(0); + //init the transducerer + transducer.init(); + //start the process + Regressor.process(transducer); + //retrieve the output + StatisticalType st = transducer.getOutput(); + System.out.println("st:"+((PrimitiveType)st).getContent()); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/tests/TestsMetaInfo.java b/src/main/java/org/gcube/dataanalysis/executor/tests/TestsMetaInfo.java new file mode 100644 index 0000000..386e41b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/tests/TestsMetaInfo.java @@ -0,0 +1,172 @@ +package org.gcube.dataanalysis.executor.tests; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.ecoengine.datatypes.StatisticalType; +import org.gcube.dataanalysis.ecoengine.interfaces.ComputationalAgent; +import org.gcube.dataanalysis.ecoengine.interfaces.Generator; +import org.gcube.dataanalysis.ecoengine.processing.factories.ClusterersFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.EvaluatorsFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.GeneratorsFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.ModelersFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.ProcessorsFactory; +import org.gcube.dataanalysis.ecoengine.processing.factories.TransducerersFactory; + +public class TestsMetaInfo { + /** + * example of parallel processing on a single machine the procedure will generate a new table for a distribution on suitable species + * + */ + +public static void main(String[] args) throws Exception { + + System.out.println("***TEST 1 - Get Generation Algorithm Parameters***"); + List map; +// List map = GeneratorsFactory.getAlgorithmParameters("./cfg/","DUMMY"); +// System.out.println("input for DUMMY algorithm: "+map); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + + StatisticalType type = GeneratorsFactory.getAlgorithmOutput("./cfg/","AQUAMAPS_SUITABLE"); + System.out.println("output for AQUAMAPS_SUITABLE algorithm: "+type); + type = ModelersFactory.getModelOutput("./cfg/","HSPEN"); + System.out.println("output for HSPEN algorithm: "+type); + type = EvaluatorsFactory.getEvaluatorOutput("./cfg/","HRS"); + System.out.println("output for HRS algorithm: "+type); + type = ClusterersFactory.getClustererOutput("./cfg/","DBSCAN"); + System.out.println("output for DBSCAN algorithm: "+type); + type = TransducerersFactory.getTransducerOutput(config,"BIOCLIMATE_HSPEC"); + System.out.println("output for BIOCLIMATE_HSPEC algorithm: "+type); + type = TransducerersFactory.getTransducerOutput(config,"BIONYM_LOCAL"); + System.out.println("output for BIONYM_LOCAL algorithm: "+type); + + map = GeneratorsFactory.getAlgorithmParameters("./cfg/","AQUAMAPS_SUITABLE"); + System.out.println("input for AQUAMAPS_SUITABLE algorithm: "+map); + + System.out.println("\n***TEST 2 - Get Generator***"); + Generator g = GeneratorsFactory.getGenerator(testConfig()); + System.out.println("Found generator "+g); + + System.out.println("\n***TEST 3 - Get All Generation Algorithms ***"); + System.out.println("Algs: "+GeneratorsFactory.getProbabilityDistributionAlgorithms("./cfg/")); + + System.out.println("\n***TEST 4 - Get All Generators ***"); + System.out.println("Gens: "+GeneratorsFactory.getAllGenerators("./cfg/")); + + System.out.println("\n***TEST 5 - Get All Models to be trained ***"); + System.out.println("Models: "+ModelersFactory.getModels("./cfg/")); + + System.out.println("\n***TEST 6 - Get All Modelers ***"); + System.out.println("Models: "+ModelersFactory.getModelers("./cfg/")); + + System.out.println("\n***TEST 7- Get Model parameters ***"); + map = ModelersFactory.getModelParameters("./cfg/","HSPEN"); + System.out.println("input for HSPEN algorithm: "+map); + + System.out.println("\n***TEST 8- Get Database Default Parameters***"); + map = ProcessorsFactory.getDefaultDatabaseConfiguration("./cfg/"); + System.out.println("Database Default Values: "+map); + + System.out.println("\n***TEST 9- Get All Evaluators ***"); + System.out.println("Database Default Values: "+EvaluatorsFactory.getAllEvaluators("./cfg/")); + + System.out.println("\n***TEST 10- Get Evaluator Parameters ***"); + map = EvaluatorsFactory.getEvaluatorParameters("./cfg/","QUALITY_ANALYSIS"); + System.out.println("Database Default Values: "+map); + + System.out.println("\n***TEST 11- Get Evaluators with a config***"); + List eval = EvaluatorsFactory.getEvaluators(testConfigEvaluator()); + System.out.println("Database Default Values: "+eval); + + System.out.println("\n***TEST 12- Get All Supported features***"); + + HashMap> features = ProcessorsFactory.getAllFeatures(config); + System.out.println("Database Default Values: "+features); + + System.out.println("\n***TEST 13- Get All Clusterers***"); + System.out.println("All Clusterers: "+ClusterersFactory.getAllClusterers("./cfg/")); + + System.out.println("\n***TEST 14- Get Clusterer Parameters ***"); + map = ClusterersFactory.getClustererParameters("./cfg/","DBSCAN"); + System.out.println("Clusterers Params: "+map); + + System.out.println("\n***TEST 15- Get Clusterers with a config***"); +// List clus = ClusterersFactory.getClusterers(testConfigClusterer()); +// System.out.println("Clusterers list: "+clus); + + System.out.println("\n***TEST 16- Get All Transducerers***"); + System.out.println("All Transducers: "+TransducerersFactory.getAllTransducerers(config)); + + System.out.println("\n***TEST 17- Get Transducerers Parameters ***"); + map = TransducerersFactory.getTransducerParameters(config,"BIOCLIMATE_HSPEC"); + System.out.println("Transducerers Params: "+map); + + System.out.println("\n***TEST 18- Get Transducerers with a config***"); +// List trans = TransducerersFactory.getTransducerers(testConfigTrans()); +// System.out.println("Transducerers list: "+trans); + + + System.out.println("\n***TEST 19- Get Agent Description***"); + String desc = ClusterersFactory.getDescription("./cfg/","DBSCAN"); + System.out.println("DESCRIPTION: "+desc); + + System.out.println("\n***TEST 20- Get USER perspective***"); + Map m = ProcessorsFactory.getAllFeaturesUser(config); + System.out.println("USER PERSPECTIVE: "+m); + + + int cores = Runtime.getRuntime().availableProcessors(); + System.out.println("Number of cores: "+cores); +} + + + private static AlgorithmConfiguration testConfigTrans() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(1); + + config.setAgent("TESTTRANS"); + + return config; + } + + private static AlgorithmConfiguration testConfigClusterer() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(1); + + config.setAgent("DBSCAN"); + + return config; + } + + private static AlgorithmConfiguration testConfigEvaluator() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(2); + + config.setAgent("QUALITY_ANALYSIS"); + + return config; + } + + private static AlgorithmConfiguration testConfig() { + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setConfigPath("./cfg/"); + config.setPersistencePath("./"); + config.setNumberOfResources(2); + config.setModel("TEST"); + + config.setAgent("SIMPLE_LOCAL"); + + return config; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/IfraRetrieval.java b/src/main/java/org/gcube/dataanalysis/executor/util/IfraRetrieval.java new file mode 100644 index 0000000..738f875 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/util/IfraRetrieval.java @@ -0,0 +1,30 @@ +package org.gcube.dataanalysis.executor.util; + +import java.util.ArrayList; +import java.util.List; + +import org.gcube.common.resources.gcore.ServiceEndpoint; +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.icclient.ICFactory; + +public class IfraRetrieval { + + public static List retrieveAddresses(String Category, String scope, String exclude) { + if (scope == null || scope.length() == 0) + return new ArrayList(); + +// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Setting Scope to " + scope+" and executing query"); + ScopeProvider.instance.set(scope); + + SimpleQuery query = ICFactory.queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Category/text() eq '" + Category + "'").addCondition("$resource/Profile[Name[not(contains(., '" + exclude + "'))]]").setResult("$resource/Profile/AccessPoint/Interface/Endpoint/text()"); + DiscoveryClient client = ICFactory.client(); + List addresses = client.submit(query); +// AnalysisLogger.getLogger().debug("RetrieveAddressesFromInfra->Query to IS finished"); + return addresses; + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/RScriptsManager.java b/src/main/java/org/gcube/dataanalysis/executor/util/RScriptsManager.java new file mode 100644 index 0000000..0257150 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/util/RScriptsManager.java @@ -0,0 +1,289 @@ +package org.gcube.dataanalysis.executor.util; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.UUID; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; +import org.gcube.dataanalysis.executor.scripts.OSCommand; + +public class RScriptsManager { + + public float status = 0; + public String currentOutputURL; + Process process; + + public String getCurrentOutputURL() { + return currentOutputURL; + } + + public void setCurrentOutputURL(String currentOutputURL) { + this.currentOutputURL = currentOutputURL; + } + + public String currentOutputFileName; + + public String getCurrentOutputFileName() { + return currentOutputFileName; + } + + public void setCurrentOutputFileName(String currentOutputFileName) { + this.currentOutputFileName = currentOutputFileName; + } + + public float getStatus() { + return status; + } + + public void setStatus(float status) { + this.status = status; + } + + public static void substituteStringInFile(String file, String newFile, String s, String sub) throws Exception { + BufferedReader br = new BufferedReader(new FileReader(new File(file))); + BufferedWriter bw = new BufferedWriter(new FileWriter(new File(newFile))); + String line = br.readLine(); + + while (line != null) { + int idx = line.indexOf(s); + if (idx >= 0) { + line = line.replace(s, sub); + } + bw.write(line + "\n"); + line = br.readLine(); + } + + br.close(); + bw.close(); + } + + public static void printRConsole(Process process) { + try { + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = br.readLine(); + StringBuffer sb = new StringBuffer(); + while (line != null) { + line = br.readLine(); + sb.append(line + "\n"); + } + AnalysisLogger.getLogger().debug(sb); + + AnalysisLogger.getLogger().debug("---ERRORS---"); + br = new BufferedReader(new InputStreamReader(process.getErrorStream())); + line = br.readLine(); + sb = new StringBuffer(); + + while (line != null) { + line = br.readLine(); + sb.append(line + "\n"); + } + AnalysisLogger.getLogger().debug(sb); + AnalysisLogger.getLogger().debug("---END OF ERRORS---"); + } catch (Exception e) { + AnalysisLogger.getLogger().debug("---END BY PROCESS INTERRUPTION---"); + } + } + + public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap inputParameters, String defaultInputFile, String defaultOutputFile, HashMap codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage) throws Exception { + executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, codeInjections, mustReturnAFile, uploadOutputOnStorage, true); + } + + public void executeRScript(AlgorithmConfiguration config, String scriptName, String inputFileURL, HashMap inputParameters, String defaultInputFile, String defaultOutputFile, HashMap codeInjections, boolean mustReturnAFile, boolean uploadOutputOnStorage, boolean deletefiles) throws Exception { + List tempfiles = new ArrayList(); + try { + status = 0; + String sandboxFolder = config.getConfigPath(); + String scriptPath = new File(sandboxFolder, scriptName).getAbsolutePath(); + // String originalScriptPath = scriptPath; + String preparedScriptPath = null; + + String owner = config.getParam("ServiceUserName"); + if (owner == null) + owner = "ecological.engine"; + + String scope = config.getGcubeScope(); + if (scope == null) + scope = ScopeProvider.instance.get(); + + AnalysisLogger.getLogger().debug("Current User: " + owner); + AnalysisLogger.getLogger().debug("Current Scope: " + scope); + AnalysisLogger.getLogger().debug("Sandbox Folder: " + sandboxFolder); + AnalysisLogger.getLogger().debug("Script : " + scriptPath); + AnalysisLogger.getLogger().debug("Prepared Script : " + preparedScriptPath); + + try { + OSCommand.ExecuteGetLine("ls", null); + OSCommand.ExecuteGetLine("pwd", null); + OSCommand.ExecuteGetLine("chmod +x * | whoami", null); + } catch (Throwable e) { + AnalysisLogger.getLogger().debug("Cannot execute SO commands"); + } + + if (codeInjections != null) { + for (String toSubstitute : codeInjections.keySet()) { + String substitution = codeInjections.get(toSubstitute); + AnalysisLogger.getLogger().debug("Substituting : " + toSubstitute + " with " + substitution); + preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath(); + substituteStringInFile(scriptPath, preparedScriptPath, toSubstitute, substitution); + AnalysisLogger.getLogger().debug("New Script name: " + preparedScriptPath+" exists: "+new File(preparedScriptPath).exists()); + tempfiles.add(preparedScriptPath); + scriptPath = preparedScriptPath; + } + } + + String newInputFile = new File(sandboxFolder, UUID.randomUUID() + defaultInputFile).getAbsolutePath().replace("\\", "/"); + String newOutputFile = UUID.randomUUID().toString() + defaultOutputFile; + int extension = defaultOutputFile.lastIndexOf("."); + if (extension > 0) + newOutputFile = defaultOutputFile.substring(0, extension) + "_" + UUID.randomUUID().toString() + "." + defaultOutputFile.substring(extension + 1); + + String newOutputFilePath = new File(sandboxFolder, newOutputFile).getAbsolutePath().replace("\\", "/"); + + tempfiles.add(newInputFile); + // tempfiles.add(newOutputFilePath); + + AnalysisLogger.getLogger().debug("New Input File is " + newInputFile); + AnalysisLogger.getLogger().debug("New Output File is " + newOutputFilePath); + + if (inputFileURL.trim().length() > 0) { + AnalysisLogger.getLogger().debug("Substituting standard Input"); + preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath(); + substituteStringInFile(scriptPath, preparedScriptPath, defaultInputFile, newInputFile); + } else { + if ((preparedScriptPath==null) || (preparedScriptPath.trim().length()==0)) + preparedScriptPath = new File(sandboxFolder, scriptName).getAbsolutePath(); + } + + tempfiles.add(preparedScriptPath); + + scriptPath = preparedScriptPath; + AnalysisLogger.getLogger().debug("Substituting standard Output"); + preparedScriptPath = new File(sandboxFolder, UUID.randomUUID() + scriptName).getAbsolutePath(); + substituteStringInFile(scriptPath, preparedScriptPath, defaultOutputFile, newOutputFilePath); + + tempfiles.add(preparedScriptPath); + + scriptPath = preparedScriptPath; + AnalysisLogger.getLogger().debug("Creating local file from remote file"); + if (inputFileURL.startsWith("http:") || inputFileURL.startsWith("smp:")) { + StorageUtils.downloadInputFile(inputFileURL, newInputFile); + } else if (inputFileURL.trim().length() > 0) { + OSCommand.FileCopy(inputFileURL, newInputFile); + } + + AnalysisLogger.getLogger().debug("Executing R script " + scriptPath); + + status = 10; + + // run the R code + process = Runtime.getRuntime().exec("R --no-save"); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(process.getOutputStream())); + for (String inputparam : inputParameters.keySet()) { + String value = inputParameters.get(inputparam); + bw.write(inputparam + "<-" + value + "\n"); + } + bw.write("source('" + scriptPath.replace("\\", "/") + "')\n"); + // bw.write("source('" + originalScriptPath.replace("\\", "/") + "')\n"); + bw.write("q()\n"); + bw.close(); + printRConsole(process); + process.destroy(); + + if ((new File(newOutputFilePath)).exists()) { + + if (uploadOutputOnStorage) { + AnalysisLogger.getLogger().debug("Found output file ... saving on the workspace"); + String outputURL = StorageUtils.uploadFilesOnStorage(scope, owner, sandboxFolder, newOutputFile); + currentOutputURL = outputURL; + AnalysisLogger.getLogger().debug("Output URL is " + outputURL); + } + currentOutputFileName = newOutputFilePath; + AnalysisLogger.getLogger().debug("Output File is " + currentOutputFileName); + + } else if (!mustReturnAFile) { + AnalysisLogger.getLogger().debug("No file detected as output"); + currentOutputFileName = null; + currentOutputURL = null; + } else + throw new Exception("The script did not return an output - an error occurred"); + + AnalysisLogger.getLogger().debug("The procedure was successful"); + status = 100f; + } catch (Exception e) { + e.printStackTrace(); + AnalysisLogger.getLogger().debug("The R Script generated an error! " + e.getLocalizedMessage()); + throw e; + } finally { + if (deletefiles) { + AnalysisLogger.getLogger().debug("Deleting temp files"); + + for (String file : tempfiles) { + boolean deleted = new File(file).delete(); + AnalysisLogger.getLogger().debug("Deleting " + file + " " + deleted); + + } + + AnalysisLogger.getLogger().debug("Done: Deleted temp files"); + } + } + } + + public void stop() { + if (process != null) { + try { + process.destroy(); + } catch (Exception e) { + + } + } + } + + public static void main1(String[] args) throws Exception { + String url = StorageUtils.uploadFilesOnStorage("/gcube/devsec/devVRE", "ecological.engine", "C:\\Users\\coro\\Documents\\", "tacsatmini.csv"); + System.out.println(url); + } + + public static void main(String[] args) throws Exception { + + RScriptsManager scriptmanager = new RScriptsManager(); + AnalysisLogger.setLogger("./cfg/" + AlgorithmConfiguration.defaultLoggerFile); + AlgorithmConfiguration config = new AlgorithmConfiguration(); + config.setGcubeScope("/gcube/devsec/devVRE"); + config.setPersistencePath("./PARALLEL_PROCESSING"); + + String scriptName = "interpolateTacsat.r"; + String inputFileURL = "smp://tacsatmini.csv?5ezvFfBOLqa2CUPC0velBUnqe0zlyBOiHB8b4B4yi0tsfi7ohCndPEZuRB/t7SXwAms8bk8KwvyHVLeQ2bBnUSnXP87yjTYTEFaZxxzNE/tmBefs5CNO3iRTeJGE8NDJ1bwVGRS//+4QELLbyLZ/GesZiduaN+bqrNOs/9/AGgA3Pq24H+aXe5suWD31Sxwu"; + String defaultInputFile = "tacsat.csv"; + String defaultOutputFile = "tacsat_interpolated.csv"; + + LinkedHashMap inputParameters = new LinkedHashMap(); + inputParameters.put("interval", "120"); + inputParameters.put("margin", "10"); + inputParameters.put("res", "100"); + inputParameters.put("method", "\"cHs\""); + inputParameters.put("fm", "0.5"); + inputParameters.put("distscale", "20"); + inputParameters.put("sigline", "0.2"); + inputParameters.put("minspeedThr", "2"); + inputParameters.put("maxspeedThr", "6"); + inputParameters.put("st", "c(minspeedThr,maxspeedThr)"); + inputParameters.put("headingAdjustment", "0"); + inputParameters.put("fast", "TRUE"); + inputParameters.put("npoints", "10"); + inputParameters.put("equalDist", "TRUE"); + + scriptmanager.executeRScript(config, scriptName, inputFileURL, inputParameters, defaultInputFile, defaultOutputFile, null, true, true); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java b/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java new file mode 100644 index 0000000..296f44c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/executor/util/StorageUtils.java @@ -0,0 +1,84 @@ +package org.gcube.dataanalysis.executor.util; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; + +import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.contentmanagement.blobstorage.service.IClient; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.contentmanager.storageclient.model.protocol.smp.Handler; +import org.gcube.contentmanager.storageclient.wrapper.AccessType; +import org.gcube.contentmanager.storageclient.wrapper.MemoryType; +import org.gcube.contentmanager.storageclient.wrapper.StorageClient; +import org.gcube.dataanalysis.ecoengine.configuration.AlgorithmConfiguration; + +public class StorageUtils { + + public static void downloadInputFile(String fileurl, String destinationFile) throws Exception{ + try { + Handler.activateProtocol(); + URL smpFile = new URL(fileurl); + URLConnection uc = (URLConnection) smpFile.openConnection(); + InputStream is = uc.getInputStream(); + AnalysisLogger.getLogger().debug("GenericWorker-> Retrieving from " + fileurl + " to :" + destinationFile); + inputStreamToFile(is, destinationFile); + is.close(); + } catch (Exception e) { + throw e; + } + } + + public static void inputStreamToFile(InputStream is, String path) throws FileNotFoundException, IOException { + FileOutputStream out = new FileOutputStream(new File(path)); + byte buf[] = new byte[1024]; + int len = 0; + while ((len = is.read(buf)) > 0) + out.write(buf, 0, len); + out.close(); + } + + public static String uploadFilesOnStorage(String scope, String user, String localFolder, String file) throws Exception { + try { + ScopeProvider.instance.set(scope); + AnalysisLogger.getLogger().info("Loading file on scope: " + scope); + IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient(); + String remotef = "/"+file; + client.put(true).LFile(new File(localFolder,file).getAbsolutePath()).RFile(remotef); + String url = client.getUrl().RFile(remotef); + AnalysisLogger.getLogger().info("Loading finished"); + System.gc(); + return url; + } catch (Exception e) { + AnalysisLogger.getLogger().info("Error in uploading file: " + e.getLocalizedMessage()); + throw e; + } + } + + + public static void downloadFilefromStorage(String scope, String user, String localFolder, String file) throws Exception { + try { + ScopeProvider.instance.set(scope); + AnalysisLogger.getLogger().info("Retrieving file on scope: " + scope); + IClient client = new StorageClient(AlgorithmConfiguration.StatisticalManagerClass, AlgorithmConfiguration.StatisticalManagerService, user, AccessType.SHARED, MemoryType.VOLATILE).getClient(); + String remotef = "/"+file; + client.get().LFile(new File(localFolder,file).getAbsolutePath()).RFile(remotef); + AnalysisLogger.getLogger().info("Retrieving finished"); + System.gc(); + } catch (Exception e) { + AnalysisLogger.getLogger().info("Error in retrieving file: " + e.getLocalizedMessage()); + throw e; + } + } + + + public static void main(String args[]) throws Exception{ + + uploadFilesOnStorage("/gcube", "CMSY", "./", "tacsat.csv"); + downloadFilefromStorage("/gcube", "CMSY", "./PARALLEL_PROCESSING", "tacsat.csv"); + } +} diff --git a/src/main/resources/cache/cache.ccf b/src/main/resources/cache/cache.ccf new file mode 100644 index 0000000..c2fbf50 --- /dev/null +++ b/src/main/resources/cache/cache.ccf @@ -0,0 +1,19 @@ +# DEFAULT CACHE REGION +jcs.default=DC +jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes +jcs.default.cacheattributes.MaxObjects=1000 +jcs.default.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache +jcs.default.cacheattributes.UseMemoryShrinker=true +jcs.default.cacheattributes.MaxMemoryIdleTimeSeconds=3600 +jcs.default.cacheattributes.ShrinkerIntervalSeconds=60 +jcs.default.cacheattributes.MaxSpoolPerRun=500 +jcs.default.elementattributes=org.apache.commons.jcs.engine.ElementAttributes +jcs.default.elementattributes.IsEternal=false +jcs.default.elementattributes.MaxLifeSeconds=21600 +jcs.default.elementattributes.IdleTime=1800 +jcs.default.elementattributes.IsSpool=true +jcs.default.elementattributes.IsRemote=true +jcs.default.elementattributes.IsLateral=true +jcs.auxiliary.DC=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory +jcs.auxiliary.DC.attributes=org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes +jcs.auxiliary.DC.attributes.DiskPath=./ \ No newline at end of file diff --git a/tacsat.csv b/tacsat.csv new file mode 100644 index 0000000..f3dd20a --- /dev/null +++ b/tacsat.csv @@ -0,0 +1,1001 @@ +"VE_COU","VE_REF","SI_LATI","SI_LONG","SI_DATE","SI_TIME","SI_SP","SI_HE","SI_DATIM" +"658237","Atlantis","10",51.4477244459512,3.58373072574753,"04/05/1800","13:32:00",0,25,1800-05-04 13:32:00 +"658238","Atlantis","10",51.4406743343966,3.58384667526838,"04/05/1800","15:28:00",0,25,1800-05-04 15:28:00 +"662888","Atlantis","10",51.4407431283058,3.59552922427561,"04/05/1800","17:22:00",0,25,1800-05-04 17:22:00 +"662889","Atlantis","10",51.4431543027703,3.58686242580321,"04/05/1800","19:18:00",0,25,1800-05-04 19:18:00 +"662890","Atlantis","10",51.4435144856712,3.58694753077719,"04/05/1800","21:12:00",0,25,1800-05-04 21:12:00 +"662891","Atlantis","10",51.4481111547514,3.59395834521763,"04/05/1800","23:08:00",0,25,1800-05-04 23:08:00 +"662892","Atlantis","10",51.4474296509796,3.59117479477078,"05/05/1800"," 1:04:00",0,111,1800-05-05 01:04:00 +"662893","Atlantis","10",51.6214494813131,3.19755943488237,"05/05/1800"," 2:58:00",12,292,1800-05-05 02:58:00 +"662894","Atlantis","10",51.6731934304331,3.17657612829283,"05/05/1800"," 4:54:00",2,43,1800-05-05 04:54:00 +"662895","Atlantis","10",51.6943234914839,3.22908421075065,"05/05/1800"," 6:48:00",2,294,1800-05-05 06:48:00 +"662896","Atlantis","10",51.6560659951191,3.16098873873707,"05/05/1800"," 8:44:00",6,144,1800-05-05 08:44:00 +"662897","Atlantis","10",51.6070417380915,3.08350550274644,"05/05/1800","10:40:00",7,248,1800-05-05 10:40:00 +"662898","Atlantis","10",51.576919331173,2.98779570994805,"05/05/1800","14:30:00",6,228,1800-05-05 14:30:00 +"662899","Atlantis","10",51.5465515189292,2.87639516843669,"05/05/1800","16:24:00",7,50,1800-05-05 16:24:00 +"662900","Atlantis","10",51.6061289456477,2.97909264942538,"05/05/1800","18:20:00",7,330,1800-05-05 18:20:00 +"662901","Atlantis","10",51.6016501706596,3.0204385781968,"05/05/1800","20:16:00",6,58,1800-05-05 20:16:00 +"662902","Atlantis","10",51.5731208998202,2.8929834927395,"05/05/1800","22:10:00",7,253,1800-05-05 22:10:00 +"662903","Atlantis","10",51.5931214836561,2.99407952163834,"06/05/1800","00: 6:00",7,245,1800-05-06 00:06:00 +"662904","Atlantis","10",51.5683078272836,2.93339168617967,"06/05/1800"," 2:00:00",7,330,1800-05-06 02:00:00 +"662905","Atlantis","10",51.5473403272065,2.87607746142615,"06/05/1800"," 3:56:00",7,55,1800-05-06 03:56:00 +"662906","Atlantis","10",51.5662122405847,2.93659450235404,"06/05/1800"," 5:52:00",7,83,1800-05-06 05:52:00 +"662907","Atlantis","10",51.6299493680494,3.036660480747,"06/05/1800"," 7:46:00",7,59,1800-05-06 07:46:00 +"662908","Atlantis","10",51.6359223276689,3.09941987676174,"06/05/1800"," 9:42:00",6,116,1800-05-06 09:42:00 +"662909","Atlantis","10",51.5344633132401,2.87679365528095,"06/05/1800","11:36:00",7,228,1800-05-06 11:36:00 +"662910","Atlantis","10",51.511504783609,2.31121867679339,"06/05/1800","13:32:00",14,271,1800-05-06 13:32:00 +"662911","Atlantis","10",51.4578128865957,1.82871440000553,"06/05/1800","15:28:00",6,172,1800-05-06 15:28:00 +"662912","Atlantis","10",51.5112028607992,1.8587873764392,"06/05/1800","17:22:00",7,25,1800-05-06 17:22:00 +"662913","Atlantis","10",51.6795650937799,1.95230285054259,"06/05/1800","19:18:00",7,20,1800-05-06 19:18:00 +"662914","Atlantis","10",51.695745846319,1.92481063032057,"06/05/1800","21:12:00",7,102,1800-05-06 21:12:00 +"662915","Atlantis","10",51.5305358059094,1.86397848288994,"06/05/1800","23:08:00",7,204,1800-05-06 23:08:00 +"662916","Atlantis","10",51.4609367726953,1.83341799937282,"07/05/1800"," 1:04:00",6,12,1800-05-07 01:04:00 +"662917","Atlantis","10",51.4061924462626,1.80183325022925,"07/05/1800"," 2:58:00",6,191,1800-05-07 02:58:00 +"662918","Atlantis","10",51.4650244918605,1.83690901399869,"07/05/1800"," 4:54:00",7,12,1800-05-07 04:54:00 +"662919","Atlantis","10",51.5882227651426,2.02424114330858,"07/05/1800"," 6:48:00",13,123,1800-05-07 06:48:00 +"662920","Atlantis","10",51.4148697609562,2.18524577266816,"07/05/1800"," 8:44:00",6,223,1800-05-07 08:44:00 +"662921","Atlantis","10",51.4457611120637,2.22457806055807,"07/05/1800","12:34:00",7,218,1800-05-07 12:34:00 +"662922","Atlantis","10",51.4589082118147,2.2554634172162,"07/05/1800","14:30:00",9,301,1800-05-07 14:30:00 +"662923","Atlantis","10",51.4187489186181,2.19479238274507,"07/05/1800","16:24:00",6,215,1800-05-07 16:24:00 +"662924","Atlantis","10",51.5047392145819,2.39375767114479,"07/05/1800","18:20:00",11,138,1800-05-07 18:20:00 +"662925","Atlantis","10",51.4934193430715,2.42827651779447,"08/05/1800"," 2:00:00",7,229,1800-05-08 02:00:00 +"662926","Atlantis","10",51.4012685797266,2.24624043547455,"08/05/1800"," 3:56:00",6,38,1800-05-08 03:56:00 +"662927","Atlantis","10",51.478767084531,2.40182553862966,"08/05/1800"," 5:52:00",5,230,1800-05-08 05:52:00 +"662928","Atlantis","10",51.4856400817088,2.40774057370703,"08/05/1800"," 7:46:00",7,43,1800-05-08 07:46:00 +"662929","Atlantis","10",51.5109846706307,2.51050425128173,"08/05/1800"," 9:42:00",6,203,1800-05-08 09:42:00 +"662930","Atlantis","10",51.3685016771392,2.30006208344549,"08/05/1800","11:36:00",6,191,1800-05-08 11:36:00 +"662931","Atlantis","10",51.4594513731385,2.34565509427059,"08/05/1800","13:32:00",7,236,1800-05-08 13:32:00 +"662932","Atlantis","10",51.4663128634985,2.3640575841507,"08/05/1800","17:22:00",7,56,1800-05-08 17:22:00 +"662933","Atlantis","10",51.3029056911548,2.4005533228144,"08/05/1800","19:18:00",7,37,1800-05-08 19:18:00 +"662934","Atlantis","10",51.3948131802687,2.64025816989597,"08/05/1800","21:12:00",7,74,1800-05-08 21:12:00 +"662935","Atlantis","10",51.4710751335397,2.92117160728946,"08/05/1800","23:08:00",11,136,1800-05-08 23:08:00 +"662936","Atlantis","10",51.4205238452363,3.47438484901283,"09/05/1800"," 1:04:00",11,82,1800-05-09 01:04:00 +"662937","Atlantis","10",51.4485953284753,3.59852260938287,"09/05/1800"," 2:58:00",0,15,1800-05-09 02:58:00 +"662938","Atlantis","10",51.4467330461042,3.58989608570375,"09/05/1800"," 4:54:00",0,15,1800-05-09 04:54:00 +"662939","Atlantis","10",51.4472292818362,3.5884271329958,"09/05/1800"," 6:48:00",0,15,1800-05-09 06:48:00 +"662940","Atlantis","10",51.4471316511449,3.58982776055485,"09/05/1800"," 8:44:00",0,15,1800-05-09 08:44:00 +"662941","Atlantis","10",51.4484958911571,3.58411605070718,"09/05/1800","12:34:00",0,15,1800-05-09 12:34:00 +"667604","Atlantis","10",51.4437598910858,3.583766422024,"11/05/1800","14:30:00",0,15,1800-05-11 14:30:00 +"667605","Atlantis","10",51.4416034219647,3.5851950481059,"11/05/1800","16:24:00",0,15,1800-05-11 16:24:00 +"667606","Atlantis","10",51.4472780093341,3.59583351112343,"11/05/1800","18:20:00",0,15,1800-05-11 18:20:00 +"667607","Atlantis","10",51.4445924110152,3.59097028739192,"11/05/1800","20:16:00",0,15,1800-05-11 20:16:00 +"667608","Atlantis","10",51.4429707641923,3.59499000045843,"11/05/1800","22:10:00",0,15,1800-05-11 22:10:00 +"667609","Atlantis","10",51.4448838578933,3.59818694050796,"12/05/1800","00: 6:00",0,15,1800-05-12 00:06:00 +"667610","Atlantis","10",51.4455564921806,3.55852451810613,"12/05/1800"," 2:00:00",14,311,1800-05-12 02:00:00 +"667611","Atlantis","10",51.6548137522223,3.14251614461281,"12/05/1800"," 5:52:00",7,218,1800-05-12 05:52:00 +"667612","Atlantis","10",51.5775022460828,2.90554459167179,"12/05/1800"," 9:42:00",7,60,1800-05-12 09:42:00 +"667613","Atlantis","10",51.6453192323837,2.95586224767007,"12/05/1800","11:36:00",7,336,1800-05-12 11:36:00 +"667614","Atlantis","10",51.911236426278,2.95896811183542,"12/05/1800","13:32:00",13,349,1800-05-12 13:32:00 +"667615","Atlantis","10",51.9453403444826,2.94993140023574,"12/05/1800","13:42:00",13,342,1800-05-12 13:42:00 +"667616","Atlantis","10",52.0593325713198,2.73492714592721,"12/05/1800","15:28:00",7,309,1800-05-12 15:28:00 +"667617","Atlantis","10",52.0947284897235,2.43497160246968,"12/05/1800","17:22:00",6,262,1800-05-12 17:22:00 +"667618","Atlantis","10",51.9565004344275,2.27307889834326,"12/05/1800","19:18:00",14,216,1800-05-12 19:18:00 +"667619","Atlantis","10",51.6590415419177,1.9370064862268,"12/05/1800","21:12:00",6,196,1800-05-12 21:12:00 +"667620","Atlantis","10",51.4842405747063,1.84669348298386,"12/05/1800","23:08:00",5,197,1800-05-12 23:08:00 +"667621","Atlantis","10",51.5091452060263,1.85640532855037,"13/05/1800"," 1:04:00",7,26,1800-05-13 01:04:00 +"667622","Atlantis","10",51.4929940155745,1.83951853084937,"13/05/1800"," 2:58:00",6,204,1800-05-13 02:58:00 +"667623","Atlantis","10",51.4471113794856,1.81924429896846,"13/05/1800"," 4:54:00",6,190,1800-05-13 04:54:00 +"667624","Atlantis","10",51.4249639321375,1.80902313761879,"13/05/1800"," 6:48:00",5,18,1800-05-13 06:48:00 +"667625","Atlantis","10",51.4693282687338,1.84038470214791,"13/05/1800"," 8:44:00",5,10,1800-05-13 08:44:00 +"667626","Atlantis","10",51.4418218042934,1.81663903429546,"13/05/1800","10:40:00",7,11,1800-05-13 10:40:00 +"667627","Atlantis","10",51.4088726234972,2.16847054663394,"13/05/1800","12:34:00",8,50,1800-05-13 12:34:00 +"667628","Atlantis","10",51.5362964495379,2.39018587551173,"13/05/1800","14:30:00",5,304,1800-05-13 14:30:00 +"667629","Atlantis","10",51.5190567561476,2.35887411012128,"13/05/1800","16:24:00",1,341,1800-05-13 16:24:00 +"667630","Atlantis","10",51.4108731082259,2.18432660974283,"13/05/1800","18:20:00",5,52,1800-05-13 18:20:00 +"667631","Atlantis","10",51.2596449704305,2.06133533269446,"13/05/1800","20:16:00",7,218,1800-05-13 20:16:00 +"667632","Atlantis","10",51.2466653574975,2.03709894624539,"13/05/1800","22:10:00",6,234,1800-05-13 22:10:00 +"667633","Atlantis","10",51.3371365424073,2.19485791365616,"14/05/1800"," 2:00:00",7,100,1800-05-14 02:00:00 +"667634","Atlantis","10",51.3545975816706,2.50843999481108,"14/05/1800"," 3:56:00",1,77,1800-05-14 03:56:00 +"667635","Atlantis","10",51.4250464622211,2.76797837713175,"14/05/1800"," 5:52:00",6,69,1800-05-14 05:52:00 +"667636","Atlantis","10",51.4208011545157,2.72131891132705,"14/05/1800"," 7:46:00",5,78,1800-05-14 07:46:00 +"667637","Atlantis","10",51.4992059450257,2.95690686668456,"14/05/1800"," 9:42:00",8,47,1800-05-14 09:42:00 +"667638","Atlantis","10",51.5439986988604,3.03306273283996,"14/05/1800","11:36:00",7,62,1800-05-14 11:36:00 +"667639","Atlantis","10",51.5604905423173,3.07679762857966,"14/05/1800","13:32:00",6,262,1800-05-14 13:32:00 +"667640","Atlantis","10",51.5730975413276,3.0872489113044,"14/05/1800","15:28:00",7,332,1800-05-14 15:28:00 +"667641","Atlantis","10",51.5740948810196,3.08704559132829,"14/05/1800","17:22:00",6,330,1800-05-14 17:22:00 +"667642","Atlantis","10",51.5730116140666,3.06881503879931,"14/05/1800","19:18:00",6,72,1800-05-14 19:18:00 +"667643","Atlantis","10",51.5742488432918,3.07494611991662,"14/05/1800","21:12:00",5,54,1800-05-14 21:12:00 +"667644","Atlantis","10",51.562859330554,3.00578095217701,"14/05/1800","23:08:00",7,242,1800-05-14 23:08:00 +"667645","Atlantis","10",51.5095643621851,2.87637615688983,"15/05/1800"," 1:04:00",7,67,1800-05-15 01:04:00 +"667646","Atlantis","10",51.566353250348,3.03543414315209,"15/05/1800"," 2:58:00",5,241,1800-05-15 02:58:00 +"667647","Atlantis","10",51.5579811066706,3.01812346521858,"15/05/1800"," 4:54:00",7,73,1800-05-15 04:54:00 +"667648","Atlantis","10",51.5684141412373,3.0783442192059,"15/05/1800"," 6:48:00",6,212,1800-05-15 06:48:00 +"667649","Atlantis","10",51.5484115438666,3.0216581239719,"15/05/1800"," 8:44:00",2,193,1800-05-15 08:44:00 +"667650","Atlantis","10",51.5452495538853,2.9637534877751,"15/05/1800","10:40:00",5,41,1800-05-15 10:40:00 +"667651","Atlantis","10",51.5808112133988,2.85450792735163,"15/05/1800","16:24:00",7,68,1800-05-15 16:24:00 +"667652","Atlantis","10",51.7043401395855,3.2504917239761,"15/05/1800","20:16:00",2,218,1800-05-15 20:16:00 +"667653","Atlantis","10",51.4437858781703,3.60456061182543,"15/05/1800","22:10:00",0,296,1800-05-15 22:10:00 +"667654","Atlantis","10",51.4469785857573,3.59578551058099,"16/05/1800","00: 6:00",0,356,1800-05-16 00:06:00 +"667655","Atlantis","10",51.4466890485212,3.59930284180119,"16/05/1800"," 2:00:00",0,25,1800-05-16 02:00:00 +"667656","Atlantis","10",51.4394142677062,3.5838732968187,"16/05/1800"," 3:56:00",0,189,1800-05-16 03:56:00 +"667657","Atlantis","10",51.4434437164781,3.58910195771232,"16/05/1800"," 5:52:00",0,189,1800-05-16 05:52:00 +"667658","Atlantis","10",51.4482281461498,3.58585640684329,"16/05/1800"," 7:46:00",0,189,1800-05-16 07:46:00 +"667659","Atlantis","10",51.4444931621128,3.59037094653491,"16/05/1800"," 9:42:00",0,354,1800-05-16 09:42:00 +"672312","Atlantis","10",51.4498283543345,3.59520274506323,"18/05/1800","17:22:00",0,354,1800-05-18 17:22:00 +"672313","Atlantis","10",51.4417422101041,3.58684735935088,"18/05/1800","19:18:00",0,354,1800-05-18 19:18:00 +"672314","Atlantis","10",51.4505019508409,3.60046504311077,"18/05/1800","21:12:00",0,354,1800-05-18 21:12:00 +"672315","Atlantis","10",51.4386785472436,3.58724097280763,"19/05/1800"," 1:04:00",0,110,1800-05-19 01:04:00 +"672316","Atlantis","10",51.6068690898926,3.2572747477619,"19/05/1800"," 2:58:00",12,291,1800-05-19 02:58:00 +"672317","Atlantis","10",51.6367579683163,3.12798582594376,"19/05/1800"," 4:54:00",6,133,1800-05-19 04:54:00 +"672318","Atlantis","10",51.6412097306293,3.12819190019648,"19/05/1800"," 6:48:00",4,312,1800-05-19 06:48:00 +"672319","Atlantis","10",51.6585236066496,3.09356418348011,"19/05/1800"," 8:44:00",6,315,1800-05-19 08:44:00 +"672320","Atlantis","10",51.6460921823192,3.15070162191056,"19/05/1800","10:40:00",6,117,1800-05-19 10:40:00 +"672321","Atlantis","10",51.6278278068313,3.1304722739188,"19/05/1800","12:34:00",6,91,1800-05-19 12:34:00 +"672322","Atlantis","10",51.5618644743208,2.91229306133464,"19/05/1800","14:30:00",6,236,1800-05-19 14:30:00 +"672323","Atlantis","10",51.5596606578669,2.94496564493887,"19/05/1800","16:24:00",7,57,1800-05-19 16:24:00 +"672324","Atlantis","10",51.5596441497481,2.93720259791147,"19/05/1800","16:24:00",7,57,1800-05-19 16:24:00 +"672325","Atlantis","10",51.5673459282885,2.92685810656566,"19/05/1800","18:20:00",7,36,1800-05-19 18:20:00 +"672326","Atlantis","10",51.5847009616452,2.98208248426113,"19/05/1800","20:16:00",6,53,1800-05-19 20:16:00 +"672327","Atlantis","10",51.5789075907171,2.9853329268042,"19/05/1800","20:16:00",6,53,1800-05-19 20:16:00 +"672328","Atlantis","10",51.6162410738356,3.07116217014753,"19/05/1800","22:10:00",6,349,1800-05-19 22:10:00 +"672329","Atlantis","10",51.5975333355735,3.02140312454756,"20/05/1800","00: 6:00",6,57,1800-05-20 00:06:00 +"672330","Atlantis","10",51.590457122311,3.00025474218279,"20/05/1800"," 2:00:00",7,250,1800-05-20 02:00:00 +"672331","Atlantis","10",51.5546195473284,2.94235223541502,"20/05/1800"," 3:56:00",6,47,1800-05-20 03:56:00 +"672332","Atlantis","10",51.5883381729759,3.00540564278979,"20/05/1800"," 5:52:00",6,61,1800-05-20 05:52:00 +"672333","Atlantis","10",51.6664156240979,3.15514422474522,"20/05/1800"," 7:46:00",3,25,1800-05-20 07:46:00 +"672334","Atlantis","10",51.6637506215745,3.1785451447973,"20/05/1800"," 9:42:00",5,305,1800-05-20 09:42:00 +"672335","Atlantis","10",51.6670854705386,3.12850809464138,"20/05/1800","11:36:00",7,199,1800-05-20 11:36:00 +"672336","Atlantis","10",51.5858145102956,3.07904149476718,"20/05/1800","13:32:00",6,53,1800-05-20 13:32:00 +"672337","Atlantis","10",51.5789679873553,3.08257830537669,"20/05/1800","13:32:00",6,53,1800-05-20 13:32:00 +"672338","Atlantis","10",51.5661317774118,3.08022928616777,"20/05/1800","15:28:00",8,254,1800-05-20 15:28:00 +"672339","Atlantis","10",51.5655843629269,3.09136122753192,"20/05/1800","15:28:00",8,254,1800-05-20 15:28:00 +"672340","Atlantis","10",51.6065265243943,3.0880650101006,"20/05/1800","17:22:00",7,99,1800-05-20 17:22:00 +"672341","Atlantis","10",51.6822944793711,3.23132003711164,"20/05/1800","21:12:00",7,299,1800-05-20 21:12:00 +"672342","Atlantis","10",51.3531013784846,2.52557839258388,"21/05/1800"," 4:54:00",2,70,1800-05-21 04:54:00 +"672343","Atlantis","10",51.3510980467331,2.5195066522155,"21/05/1800"," 6:48:00",2,128,1800-05-21 06:48:00 +"672344","Atlantis","10",51.4057357607242,1.98011851156689,"21/05/1800"," 8:44:00",11,276,1800-05-21 08:44:00 +"672345","Atlantis","10",51.6933935687239,1.91206538324058,"21/05/1800","12:34:00",5,56,1800-05-21 12:34:00 +"672346","Atlantis","10",51.5936210588138,1.90483856583666,"21/05/1800","14:30:00",6,122,1800-05-21 14:30:00 +"672347","Atlantis","10",51.4307533462876,1.8138914814638,"21/05/1800","16:24:00",10,138,1800-05-21 16:24:00 +"672348","Atlantis","10",51.4701913939635,1.8344310724251,"21/05/1800","18:20:00",6,9,1800-05-21 18:20:00 +"672349","Atlantis","10",51.6079437457426,1.91385515465308,"21/05/1800","20:16:00",7,3,1800-05-21 20:16:00 +"672350","Atlantis","10",51.5875464249258,2.24126899376046,"21/05/1800","22:10:00",12,122,1800-05-21 22:10:00 +"672351","Atlantis","10",51.4389535234203,2.22209049706813,"22/05/1800","00: 6:00",6,217,1800-05-22 00:06:00 +"672352","Atlantis","10",51.4003563606981,2.17254899724946,"22/05/1800"," 2:00:00",6,222,1800-05-22 02:00:00 +"672353","Atlantis","10",51.4411780570857,2.22556785787735,"22/05/1800"," 3:56:00",6,44,1800-05-22 03:56:00 +"672354","Atlantis","10",51.4443513077423,2.22165709033236,"22/05/1800"," 5:52:00",5,217,1800-05-22 05:52:00 +"672355","Atlantis","10",51.4379137101243,2.22858026063163,"22/05/1800"," 7:46:00",5,214,1800-05-22 07:46:00 +"672356","Atlantis","10",51.5139608935807,2.3422037399048,"22/05/1800"," 9:42:00",6,57,1800-05-22 09:42:00 +"672357","Atlantis","10",51.5011061795512,2.43439523145277,"22/05/1800","11:36:00",7,48,1800-05-22 11:36:00 +"672358","Atlantis","10",51.4499953094176,2.33965058990195,"22/05/1800","13:32:00",6,24,1800-05-22 13:32:00 +"672359","Atlantis","10",51.3898783241338,2.24577099526953,"22/05/1800","15:28:00",3,13,1800-05-22 15:28:00 +"672360","Atlantis","10",51.5051320922324,2.44321047574468,"22/05/1800","19:18:00",7,49,1800-05-22 19:18:00 +"672361","Atlantis","10",51.5075685075247,2.44582821303233,"22/05/1800","19:18:00",7,49,1800-05-22 19:18:00 +"672362","Atlantis","10",51.545395918095,2.63815955818817,"22/05/1800","21:12:00",4,198,1800-05-22 21:12:00 +"672363","Atlantis","10",51.5958505059853,2.74833332258091,"22/05/1800","23:08:00",8,94,1800-05-22 23:08:00 +"672364","Atlantis","10",51.4163502259934,3.22519611601811,"23/05/1800"," 1:04:00",11,79,1800-05-23 01:04:00 +"672365","Atlantis","10",51.4442419000026,3.58922122503072,"23/05/1800"," 2:58:00",4,322,1800-05-23 02:58:00 +"672366","Atlantis","10",51.4487870216602,3.59106035409402,"23/05/1800"," 4:54:00",0,92,1800-05-23 04:54:00 +"672367","Atlantis","10",51.4488727705739,3.59884215049352,"23/05/1800"," 6:48:00",0,92,1800-05-23 06:48:00 +"672368","Atlantis","10",51.4435630390584,3.58929342525918,"23/05/1800"," 8:44:00",0,92,1800-05-23 08:44:00 +"676983","Atlantis","10",51.4425128327706,3.59011167568993,"23/05/1800","10:40:00",0,92,1800-05-23 10:40:00 +"676984","Atlantis","10",51.443881428598,3.58982501373068,"23/05/1800","12:34:00",0,92,1800-05-23 12:34:00 +"677008","Atlantis","10",51.4413676513359,3.5852599167116,"25/05/1800","18:20:00",0,92,1800-05-25 18:20:00 +"677009","Atlantis","10",51.4454871094599,3.58342768724915,"25/05/1800","20:16:00",0,92,1800-05-25 20:16:00 +"677010","Atlantis","10",51.4472130877082,3.58618369957339,"25/05/1800","22:10:00",0,92,1800-05-25 22:10:00 +"677011","Atlantis","10",51.4405778946774,3.59741392126679,"26/05/1800","00: 6:00",0,92,1800-05-26 00:06:00 +"677012","Atlantis","10",51.5965147950058,3.31778014119342,"26/05/1800"," 2:00:00",12,292,1800-05-26 02:00:00 +"677013","Atlantis","10",51.679514797824,3.22658743065875,"26/05/1800"," 3:56:00",7,215,1800-05-26 03:56:00 +"677014","Atlantis","10",51.6534445190164,3.15240749916621,"26/05/1800"," 5:52:00",6,203,1800-05-26 05:52:00 +"677015","Atlantis","10",51.6226765899486,3.11763341970555,"26/05/1800"," 7:46:00",7,179,1800-05-26 07:46:00 +"677016","Atlantis","10",51.5529266411574,2.91942515312974,"26/05/1800"," 9:42:00",6,27,1800-05-26 09:42:00 +"677017","Atlantis","10",51.5761613158053,2.98066639199201,"26/05/1800","11:36:00",7,51,1800-05-26 11:36:00 +"677018","Atlantis","10",51.586419825085,3.00868589759059,"26/05/1800","13:32:00",7,49,1800-05-26 13:32:00 +"677019","Atlantis","10",51.5888217586107,3.03118455032259,"26/05/1800","15:28:00",6,54,1800-05-26 15:28:00 +"677020","Atlantis","10",51.5710880040219,2.93852564639505,"26/05/1800","17:22:00",6,64,1800-05-26 17:22:00 +"677021","Atlantis","10",51.558819605655,2.96253412400465,"26/05/1800","19:18:00",6,67,1800-05-26 19:18:00 +"677022","Atlantis","10",51.5459717204808,2.89160391475074,"26/05/1800","21:12:00",1,38,1800-05-26 21:12:00 +"677023","Atlantis","10",51.5362278128476,2.89195539133157,"26/05/1800","21:12:00",NA,NA,1800-05-26 21:12:00 +"677024","Atlantis","10",51.5609061656599,2.87349012409058,"26/05/1800","23:08:00",NA,NA,1800-05-26 23:08:00 +"677025","Atlantis","10",51.5614489930412,2.87179454894923,"26/05/1800","23:08:00",6,42,1800-05-26 23:08:00 +"677026","Atlantis","10",51.7253597725076,3.01744475515373,"27/05/1800"," 1:04:00",7,69,1800-05-27 01:04:00 +"677027","Atlantis","10",51.7456746744616,3.07719044709206,"27/05/1800"," 2:58:00",6,234,1800-05-27 02:58:00 +"677028","Atlantis","10",51.7155772482995,3.12027825713158,"27/05/1800"," 4:54:00",6,235,1800-05-27 04:54:00 +"677029","Atlantis","10",51.5984315578206,2.96170603665989,"27/05/1800"," 6:48:00",6,59,1800-05-27 06:48:00 +"677030","Atlantis","10",51.6127899034694,3.01407608689554,"27/05/1800"," 8:44:00",8,246,1800-05-27 08:44:00 +"677031","Atlantis","10",51.4931282767719,2.81815157718118,"27/05/1800","10:40:00",6,257,1800-05-27 10:40:00 +"677032","Atlantis","10",51.4981504839892,2.33773900338728,"27/05/1800","12:34:00",12,276,1800-05-27 12:34:00 +"677033","Atlantis","10",51.5770034613227,1.90571926696598,"27/05/1800","14:30:00",7,26,1800-05-27 14:30:00 +"677034","Atlantis","10",51.67751587275,1.94941334740631,"27/05/1800","16:24:00",7,21,1800-05-27 16:24:00 +"677035","Atlantis","10",51.6778512750175,1.95051395815238,"27/05/1800","16:24:00",7,21,1800-05-27 16:24:00 +"677036","Atlantis","10",51.6265724248295,1.92285628780909,"27/05/1800","18:20:00",7,181,1800-05-27 18:20:00 +"677037","Atlantis","10",51.4901789905271,1.84238064859156,"27/05/1800","20:16:00",5,25,1800-05-27 20:16:00 +"677038","Atlantis","10",51.4941483656736,1.8341560415607,"27/05/1800","20:16:00",NA,NA,1800-05-27 20:16:00 +"677039","Atlantis","10",51.4925464702961,1.84628458611108,"28/05/1800"," 2:00:00",7,206,1800-05-28 02:00:00 +"677040","Atlantis","10",51.4557961362796,1.81918313868251,"28/05/1800"," 3:56:00",6,187,1800-05-28 03:56:00 +"677041","Atlantis","10",51.5044706184217,1.85029320965148,"28/05/1800"," 5:52:00",6,27,1800-05-28 05:52:00 +"677042","Atlantis","10",51.5158075350937,1.84238087548316,"28/05/1800"," 7:46:00",7,199,1800-05-28 07:46:00 +"677043","Atlantis","10",51.427995924768,1.81726581050921,"28/05/1800"," 9:42:00",6,16,1800-05-28 09:42:00 +"677044","Atlantis","10",51.4963966847067,1.85540833191294,"28/05/1800","11:36:00",7,22,1800-05-28 11:36:00 +"677045","Atlantis","10",51.6439060433051,1.92130181420967,"28/05/1800","13:32:00",7,11,1800-05-28 13:32:00 +"677046","Atlantis","10",51.6822505572815,1.93689612194802,"28/05/1800","15:28:00",6,135,1800-05-28 15:28:00 +"677047","Atlantis","10",51.6451249079532,2.0047009931244,"28/05/1800","17:22:00",13,106,1800-05-28 17:22:00 +"677048","Atlantis","10",51.5224785884791,2.19522223338578,"28/05/1800","19:18:00",6,37,1800-05-28 19:18:00 +"677049","Atlantis","10",51.5299558686926,2.20393963364977,"28/05/1800","21:14:00",7,218,1800-05-28 21:14:00 +"677050","Atlantis","10",51.5385962066553,2.21860907216091,"28/05/1800","23:08:00",6,38,1800-05-28 23:08:00 +"677051","Atlantis","10",51.4697996221241,2.41705975234602,"29/05/1800"," 1:04:00",14,90,1800-05-29 01:04:00 +"677052","Atlantis","10",51.4062104156092,3.10094109585975,"29/05/1800"," 2:58:00",13,102,1800-05-29 02:58:00 +"677053","Atlantis","10",51.4411262052413,3.59029239661433,"29/05/1800"," 4:54:00",1,268,1800-05-29 04:54:00 +"677054","Atlantis","10",51.4428056301554,3.59224467536621,"29/05/1800"," 6:48:00",0,49,1800-05-29 06:48:00 +"677055","Atlantis","10",51.4454917158089,3.59597365520615,"29/05/1800"," 8:44:00",0,49,1800-05-29 08:44:00 +"681675","Atlantis","10",51.4516734176693,3.58644658665825,"29/05/1800","10:40:00",0,49,1800-05-29 10:40:00 +"681676","Atlantis","10",51.4495323975226,3.58543850317318,"29/05/1800","12:34:00",0,49,1800-05-29 12:34:00 +"681677","Atlantis","10",51.4467095976421,3.60253330370132,"29/05/1800","14:30:00",0,49,1800-05-29 14:30:00 +"681678","Atlantis","10",51.4497009727526,3.59032916679699,"29/05/1800","16:24:00",0,49,1800-05-29 16:24:00 +"1547480","Atlantis","10",51.4471215216001,3.5874665289782,"05/09/1801","15:46:00",0,352,1801-09-05 15:46:00 +"1547481","Atlantis","10",51.4472549664276,3.59630610466376,"05/09/1801","17:42:00",0,352,1801-09-05 17:42:00 +"1547482","Atlantis","10",51.4489429760189,3.58790835902467,"05/09/1801","19:38:00",0,352,1801-09-05 19:38:00 +"1547483","Atlantis","10",51.4493196887663,3.58770060952194,"05/09/1801","21:32:00",0,352,1801-09-05 21:32:00 +"1547484","Atlantis","10",51.4445854990501,3.58647985284589,"05/09/1801","23:28:00",0,352,1801-09-05 23:28:00 +"1547485","Atlantis","10",51.4398813429773,3.59849998850748,"06/09/1801"," 1:22:00",0,114,1801-09-06 01:22:00 +"1547486","Atlantis","10",51.6479286842844,3.25952713057864,"06/09/1801"," 3:18:00",12,305,1801-09-06 03:18:00 +"1547487","Atlantis","10",51.6281376386625,3.0386842816649,"06/09/1801"," 7:08:00",7,66,1801-09-06 07:08:00 +"1547488","Atlantis","10",51.6509485563682,3.08952733156458,"06/09/1801"," 9:04:00",6,300,1801-09-06 09:04:00 +"1547489","Atlantis","10",51.6481914477092,3.14860756535083,"06/09/1801","10:58:00",5,129,1801-09-06 10:58:00 +"1547490","Atlantis","10",51.6529306280897,3.14159676210675,"06/09/1801","14:48:00",6,132,1801-09-06 14:48:00 +"1547491","Atlantis","10",51.670379133218,3.19591958882101,"06/09/1801","16:44:00",6,125,1801-09-06 16:44:00 +"1547492","Atlantis","10",51.6716838364308,3.19370284848195,"06/09/1801","16:44:00",0,58,1801-09-06 16:44:00 +"1547493","Atlantis","10",51.6817619224284,3.2123439415684,"06/09/1801","18:40:00",6,325,1801-09-06 18:40:00 +"1547494","Atlantis","10",51.6763011609525,3.1989973416226,"06/09/1801","18:40:00",6,325,1801-09-06 18:40:00 +"1547495","Atlantis","10",51.6873501892756,3.23244109593984,"06/09/1801","20:34:00",7,147,1801-09-06 20:34:00 +"1547496","Atlantis","10",51.6871790735023,3.23096863422263,"06/09/1801","20:34:00",7,147,1801-09-06 20:34:00 +"1547497","Atlantis","10",51.6851985401819,3.22443534885813,"06/09/1801","22:30:00",6,167,1801-09-06 22:30:00 +"1547498","Atlantis","10",51.6974421805986,3.25139044718072,"07/09/1801","00:26:00",7,320,1801-09-07 00:26:00 +"1547499","Atlantis","10",51.697171198735,3.23995037373155,"07/09/1801"," 2:20:00",7,81,1801-09-07 02:20:00 +"1547500","Atlantis","10",51.6978131556478,3.24973523151688,"07/09/1801"," 4:16:00",1,182,1801-09-07 04:16:00 +"1547501","Atlantis","10",51.7014432861628,3.24942585768644,"07/09/1801"," 6:10:00",4,305,1801-09-07 06:10:00 +"1547502","Atlantis","10",51.6987577115968,3.23692428721488,"07/09/1801"," 8:06:00",6,237,1801-09-07 08:06:00 +"1547503","Atlantis","10",51.7838488895763,3.29683520968538,"07/09/1801","10:02:00",6,206,1801-09-07 10:02:00 +"1547504","Atlantis","10",51.8068421097542,3.32008304503187,"07/09/1801","11:56:00",6,204,1801-09-07 11:56:00 +"1547505","Atlantis","10",51.7046374504543,3.24381653071474,"07/09/1801","13:52:00",7,207,1801-09-07 13:52:00 +"1547506","Atlantis","10",51.5438197602029,3.04950336607546,"07/09/1801","15:46:00",7,234,1801-09-07 15:46:00 +"1547507","Atlantis","10",51.4303528347453,2.74918987647444,"07/09/1801","17:42:00",6,294,1801-09-07 17:42:00 +"1547508","Atlantis","10",51.3394456307925,2.51056399023999,"07/09/1801","19:36:00",6,212,1801-09-07 19:36:00 +"1547509","Atlantis","10",51.3166115203057,2.41250956253149,"07/09/1801","21:32:00",9,350,1801-09-07 21:32:00 +"1547510","Atlantis","10",51.2819146101475,2.37883586782962,"07/09/1801","23:28:00",2,223,1801-09-07 23:28:00 +"1547511","Atlantis","10",51.2798472953723,2.37747471261769,"07/09/1801","23:28:00",2,223,1801-09-07 23:28:00 +"1547512","Atlantis","10",51.2353007429149,2.00376235204376,"08/09/1801"," 1:22:00",7,256,1801-09-08 01:22:00 +"1547513","Atlantis","10",51.2427454718272,2.01149504890945,"08/09/1801"," 3:18:00",5,322,1801-09-08 03:18:00 +"1547514","Atlantis","10",51.2019989012559,1.89096347641386,"08/09/1801"," 5:14:00",7,240,1801-09-08 05:14:00 +"1547515","Atlantis","10",51.2277650822424,1.95771768193971,"08/09/1801"," 7:08:00",7,39,1801-09-08 07:08:00 +"1547516","Atlantis","10",51.2359607488536,1.98193690547533,"08/09/1801"," 9:04:00",7,242,1801-09-08 09:04:00 +"1547517","Atlantis","10",51.2512144695586,2.03653786873538,"08/09/1801","10:58:00",7,38,1801-09-08 10:58:00 +"1547518","Atlantis","10",51.2490607190146,2.01211349516548,"08/09/1801","12:54:00",5,53,1801-09-08 12:54:00 +"1547519","Atlantis","10",51.2067375236326,1.901906176758,"08/09/1801","16:44:00",7,148,1801-09-08 16:44:00 +"1547520","Atlantis","10",51.2229271445018,1.92765690828487,"08/09/1801","18:40:00",6,60,1801-09-08 18:40:00 +"1547521","Atlantis","10",51.2217695461111,1.93127365481388,"08/09/1801","18:40:00",6,60,1801-09-08 18:40:00 +"1547522","Atlantis","10",51.2184366785004,1.941739888818,"08/09/1801","18:40:00",6,60,1801-09-08 18:40:00 +"1547523","Atlantis","10",51.3302821980892,2.16700047973357,"08/09/1801","20:34:00",6,37,1801-09-08 20:34:00 +"1547524","Atlantis","10",51.3502505001551,2.19771695313882,"08/09/1801","22:30:00",7,189,1801-09-08 22:30:00 +"1550103","Atlantis","10",51.4034635625151,2.25437117671408,"09/09/1801","00:26:00",6,80,1801-09-09 00:26:00 +"1550104","Atlantis","10",51.322973999144,2.150665619337,"09/09/1801"," 2:20:00",6,237,1801-09-09 02:20:00 +"1550105","Atlantis","10",51.222391182302,1.93384957790095,"09/09/1801"," 4:16:00",6,112,1801-09-09 04:16:00 +"1550106","Atlantis","10",51.2196073813797,1.92921392862871,"09/09/1801"," 4:16:00",6,112,1801-09-09 04:16:00 +"1550107","Atlantis","10",51.2129375260947,1.90642654551938,"09/09/1801"," 6:10:00",3,235,1801-09-09 06:10:00 +"1550108","Atlantis","10",51.2305429565869,1.92083688542153,"09/09/1801"," 8:06:00",7,88,1801-09-09 08:06:00 +"1550109","Atlantis","10",51.2666672397051,2.02879456453118,"09/09/1801","10:02:00",6,159,1801-09-09 10:02:00 +"1550110","Atlantis","10",51.3289543471597,2.1834359162394,"09/09/1801","11:56:00",7,47,1801-09-09 11:56:00 +"1550111","Atlantis","10",51.4675729378844,2.39237256994657,"09/09/1801","13:52:00",7,18,1801-09-09 13:52:00 +"1550112","Atlantis","10",51.461545520769,2.39517988363002,"09/09/1801","13:52:00",7,18,1801-09-09 13:52:00 +"1550113","Atlantis","10",51.5817346513038,2.70298299446236,"09/09/1801","15:46:00",12,71,1801-09-09 15:46:00 +"1550114","Atlantis","10",51.70327139585,3.12450265863724,"09/09/1801","17:42:00",6,56,1801-09-09 17:42:00 +"1550115","Atlantis","10",51.7465460068672,3.30236414455436,"09/09/1801","19:36:00",6,29,1801-09-09 19:36:00 +"1550116","Atlantis","10",51.7407954454753,3.22458097548038,"09/09/1801","21:32:00",6,245,1801-09-09 21:32:00 +"1550117","Atlantis","10",51.6340549074784,3.11983212189563,"09/09/1801","23:28:00",7,250,1801-09-09 23:28:00 +"1550118","Atlantis","10",51.4817744424981,3.49791612724587,"10/09/1801"," 1:22:00",12,139,1801-09-10 01:22:00 +"1550119","Atlantis","10",51.4463830043748,3.5825653932374,"10/09/1801"," 3:18:00",0,4,1801-09-10 03:18:00 +"1550120","Atlantis","10",51.4447868594038,3.59198094516899,"10/09/1801"," 5:12:00",0,4,1801-09-10 05:12:00 +"1550121","Atlantis","10",51.4470107603027,3.58766982076131,"10/09/1801"," 7:08:00",0,4,1801-09-10 07:08:00 +"1550122","Atlantis","10",51.4420946024871,3.59441434751172,"10/09/1801"," 9:04:00",0,4,1801-09-10 09:04:00 +"1550123","Atlantis","10",51.4428739119833,3.59144667406008,"10/09/1801","10:58:00",0,4,1801-09-10 10:58:00 +"1550124","Atlantis","10",51.4442425165256,3.58801649141032,"10/09/1801","12:54:00",0,4,1801-09-10 12:54:00 +"1550148","Atlantis","10",51.440633860731,3.59315317070484,"12/09/1801","16:44:00",0,4,1801-09-12 16:44:00 +"1550149","Atlantis","10",51.4435586108244,3.59003407027945,"12/09/1801","18:40:00",0,4,1801-09-12 18:40:00 +"1550150","Atlantis","10",51.443760890679,3.58849386924319,"12/09/1801","20:34:00",0,4,1801-09-12 20:34:00 +"1550151","Atlantis","10",51.4478253994952,3.596914519988,"12/09/1801","22:30:00",0,4,1801-09-12 22:30:00 +"1550152","Atlantis","10",51.4415832231264,3.58387214612588,"13/09/1801","00:26:00",0,4,1801-09-13 00:26:00 +"1550153","Atlantis","10",51.5560800412539,3.40860676246788,"13/09/1801"," 2:20:00",13,342,1801-09-13 02:20:00 +"1550154","Atlantis","10",51.6506954165152,3.10684370871633,"13/09/1801"," 4:16:00",6,142,1801-09-13 04:16:00 +"1550155","Atlantis","10",51.6603912329171,3.13684440137353,"13/09/1801"," 6:10:00",7,326,1801-09-13 06:10:00 +"1550156","Atlantis","10",51.7078825420258,3.03145686939359,"13/09/1801"," 8:06:00",6,217,1801-09-13 08:06:00 +"1550157","Atlantis","10",51.5953640259528,2.96931218023133,"13/09/1801","10:02:00",6,332,1801-09-13 10:02:00 +"1550158","Atlantis","10",51.5944646433513,2.98598895595595,"13/09/1801","10:02:00",3,2,1801-09-13 10:02:00 +"1550159","Atlantis","10",51.5903429940022,2.99568230347335,"13/09/1801","11:56:00",7,350,1801-09-13 11:56:00 +"1550160","Atlantis","10",51.593638931362,2.98919582143519,"13/09/1801","11:56:00",7,350,1801-09-13 11:56:00 +"1550161","Atlantis","10",51.6523126868028,3.10441637480073,"13/09/1801","13:52:00",7,53,1801-09-13 13:52:00 +"1550162","Atlantis","10",51.8188790104301,3.04491055795364,"13/09/1801","15:46:00",7,340,1801-09-13 15:46:00 +"1550163","Atlantis","10",52.0010131651941,2.9353805143265,"13/09/1801","17:42:00",4,184,1801-09-13 17:42:00 +"1550164","Atlantis","10",52.0411264571841,2.94222063376568,"13/09/1801","19:38:00",5,63,1801-09-13 19:38:00 +"1550165","Atlantis","10",51.9841536738188,2.89393158707861,"13/09/1801","21:32:00",6,62,1801-09-13 21:32:00 +"1550166","Atlantis","10",51.9448920345618,3.00449599343538,"13/09/1801","23:28:00",7,126,1801-09-13 23:28:00 +"1550167","Atlantis","10",51.9292196648321,3.05030322241504,"14/09/1801"," 1:22:00",5,261,1801-09-14 01:22:00 +"1550168","Atlantis","10",51.9546804678007,3.00834763032384,"14/09/1801"," 3:18:00",4,166,1801-09-14 03:18:00 +"1550169","Atlantis","10",51.9349663437787,3.06636027573794,"14/09/1801"," 5:12:00",5,194,1801-09-14 05:12:00 +"1550170","Atlantis","10",51.9503624857804,2.99984485744871,"14/09/1801"," 7:08:00",1,50,1801-09-14 07:08:00 +"1550171","Atlantis","10",51.8076397209452,3.07649357305467,"14/09/1801"," 9:04:00",6,205,1801-09-14 09:04:00 +"1550172","Atlantis","10",51.6352427168596,3.15532667065319,"14/09/1801","10:58:00",7,223,1801-09-14 10:58:00 +"1550173","Atlantis","10",51.5780256870347,3.09939626664575,"14/09/1801","12:54:00",3,158,1801-09-14 12:54:00 +"1550174","Atlantis","10",51.6252489654711,3.12505094688013,"14/09/1801","14:50:00",2,33,1801-09-14 14:50:00 +"1552741","Atlantis","10",51.758331616886,3.31563718857337,"14/09/1801","16:44:00",8,29,1801-09-14 16:44:00 +"1552742","Atlantis","10",51.9289247930767,3.3026310300082,"14/09/1801","18:40:00",8,239,1801-09-14 18:40:00 +"1552743","Atlantis","10",51.885049523632,3.29192748449184,"14/09/1801","20:34:00",6,269,1801-09-14 20:34:00 +"1552744","Atlantis","10",51.8625549719837,3.283867040121,"14/09/1801","22:30:00",1,189,1801-09-14 22:30:00 +"1552745","Atlantis","10",51.7350381040433,3.28467455265299,"15/09/1801","00:24:00",7,181,1801-09-15 00:24:00 +"1552746","Atlantis","10",51.6980518250223,3.25246431659069,"15/09/1801"," 2:20:00",6,151,1801-09-15 02:20:00 +"1552747","Atlantis","10",51.6917256179377,3.2417630917579,"15/09/1801"," 4:16:00",6,327,1801-09-15 04:16:00 +"1552748","Atlantis","10",51.6851408106345,3.25651520087663,"15/09/1801"," 6:10:00",2,64,1801-09-15 06:10:00 +"1552749","Atlantis","10",51.6880665467288,3.20573230723571,"15/09/1801"," 8:06:00",7,112,1801-09-15 08:06:00 +"1552750","Atlantis","10",51.5565556498272,3.06820806097053,"15/09/1801","10:02:00",7,240,1801-09-15 10:02:00 +"1552751","Atlantis","10",51.4365906517385,2.78492021421157,"15/09/1801","11:56:00",7,231,1801-09-15 11:56:00 +"1552752","Atlantis","10",51.3438614295041,2.31500267245341,"15/09/1801","15:46:00",6,273,1801-09-15 15:46:00 +"1552753","Atlantis","10",51.3456505737547,2.29505123374984,"15/09/1801","15:52:00",6,331,1801-09-15 15:52:00 +"1552754","Atlantis","10",51.3535146802477,2.29615095210448,"15/09/1801","15:54:00",6,335,1801-09-15 15:54:00 +"1552755","Atlantis","10",51.5142760261507,2.45296865792852,"15/09/1801","17:42:00",6,57,1801-09-15 17:42:00 +"1552756","Atlantis","10",51.5572330715698,2.52904815525655,"15/09/1801","19:38:00",6,200,1801-09-15 19:38:00 +"1552757","Atlantis","10",51.488495921168,2.40751826915518,"15/09/1801","21:32:00",7,225,1801-09-15 21:32:00 +"1552758","Atlantis","10",51.3399569089254,2.19267187903449,"15/09/1801","23:28:00",7,190,1801-09-15 23:28:00 +"1552759","Atlantis","10",51.247439417942,2.00999379438162,"16/09/1801"," 4:00:00",NA,NA,1801-09-16 04:00:00 +"1552760","Atlantis","10",51.2341361845899,2.01905191429146,"16/09/1801"," 4:10:00",5,207,1801-09-16 04:10:00 +"1552761","Atlantis","10",51.2699721660782,2.05125782146491,"16/09/1801"," 5:14:00",5.4,196,1801-09-16 05:14:00 +"1552762","Atlantis","10",51.2652179177795,2.04115732903034,"16/09/1801"," 5:14:00",7,30,1801-09-16 05:14:00 +"1552763","Atlantis","10",51.4107242681323,2.1807154715918,"16/09/1801"," 7:08:00",7,39,1801-09-16 07:08:00 +"1552764","Atlantis","10",51.4056691547669,2.19061063135974,"16/09/1801","10:58:00",6,221,1801-09-16 10:58:00 +"1552765","Atlantis","10",51.3010033387663,2.40550455520488,"16/09/1801","12:54:00",7,197,1801-09-16 12:54:00 +"1552766","Atlantis","10",51.3449685856211,2.51781653188448,"16/09/1801","14:50:00",5,216,1801-09-16 14:50:00 +"1552805","Atlantis","10",51.4457968543051,3.58957450803742,"19/09/1801","17:42:00",0,5,1801-09-19 17:42:00 +"1552806","Atlantis","10",51.4425450845039,3.59463539364189,"19/09/1801","19:36:00",0,5,1801-09-19 19:36:00 +"1552807","Atlantis","10",51.4441567413183,3.60005367420986,"19/09/1801","21:32:00",0,5,1801-09-19 21:32:00 +"1552808","Atlantis","10",51.447090309721,3.59577182918694,"19/09/1801","23:28:00",0,5,1801-09-19 23:28:00 +"1552809","Atlantis","10",51.4385658269576,3.59124963630829,"20/09/1801"," 1:22:00",8,121,1801-09-20 01:22:00 +"1552810","Atlantis","10",51.6690986812087,3.26875787262898,"20/09/1801"," 3:18:00",2,242,1801-09-20 03:18:00 +"1552811","Atlantis","10",51.6927783189821,3.22378886089847,"20/09/1801"," 5:14:00",3,54,1801-09-20 05:14:00 +"1552812","Atlantis","10",51.645804771685,3.14713836160861,"20/09/1801"," 7:08:00",2,36,1801-09-20 07:08:00 +"1552813","Atlantis","10",51.6374378545401,3.13225625028834,"20/09/1801"," 9:04:00",7,285,1801-09-20 09:04:00 +"1555385","Atlantis","10",51.6393622103091,3.10291450892761,"20/09/1801","10:58:00",0,0,1801-09-20 10:58:00 +"1555386","Atlantis","10",51.6427481339928,3.09245328405313,"20/09/1801","12:54:00",7,298,1801-09-20 12:54:00 +"1555387","Atlantis","10",51.6478532274701,3.0873692529602,"20/09/1801","14:50:00",5,301,1801-09-20 14:50:00 +"1555388","Atlantis","10",51.6268278803383,3.09525243391097,"20/09/1801","15:50:00",7,232,1801-09-20 15:50:00 +"1555389","Atlantis","10",51.6272949674968,3.07175986222364,"20/09/1801","16:44:00",6,313,1801-09-20 16:44:00 +"1555390","Atlantis","10",51.677005219867,3.14597554130386,"20/09/1801","18:40:00",4,136,1801-09-20 18:40:00 +"1555391","Atlantis","10",51.673920142578,3.23357144267764,"20/09/1801","20:34:00",6,109,1801-09-20 20:34:00 +"1555392","Atlantis","10",51.6958931647255,3.21720166679099,"20/09/1801","22:30:00",7,144,1801-09-20 22:30:00 +"1555393","Atlantis","10",51.6121485243998,3.10585340487864,"21/09/1801","00:26:00",6,267,1801-09-21 00:26:00 +"1555394","Atlantis","10",51.6081298023318,2.97478686106857,"21/09/1801"," 2:20:00",6,247,1801-09-21 02:20:00 +"1555395","Atlantis","10",51.6101169113438,2.97946308467071,"21/09/1801"," 2:20:00",6,247,1801-09-21 02:20:00 +"1555396","Atlantis","10",51.6058486282392,2.98607299769577,"21/09/1801"," 4:16:00",7,122,1801-09-21 04:16:00 +"1555397","Atlantis","10",51.5971814411571,2.94591727061663,"21/09/1801"," 6:10:00",8,74,1801-09-21 06:10:00 +"1555398","Atlantis","10",51.600600687827,3.00779928461835,"21/09/1801"," 8:06:00",5,68,1801-09-21 08:06:00 +"1555399","Atlantis","10",51.5715216041734,3.07609878732823,"21/09/1801","10:00:00",4,73,1801-09-21 10:00:00 +"1555400","Atlantis","10",51.5727340158257,3.07912814380229,"21/09/1801","11:56:00",7,149,1801-09-21 11:56:00 +"1555401","Atlantis","10",51.5285637688744,2.83894046649151,"21/09/1801","13:52:00",7,229,1801-09-21 13:52:00 +"1555402","Atlantis","10",51.5342896535024,2.86976746339165,"21/09/1801","15:46:00",7,228,1801-09-21 15:46:00 +"1555403","Atlantis","10",51.5512972140349,2.89446464026719,"21/09/1801","17:42:00",5,218,1801-09-21 17:42:00 +"1555404","Atlantis","10",51.5496101872013,2.88918378477078,"21/09/1801","19:38:00",7,53,1801-09-21 19:38:00 +"1555405","Atlantis","10",51.5692254377492,3.08719663170259,"21/09/1801","21:32:00",6,246,1801-09-21 21:32:00 +"1555406","Atlantis","10",51.6148937949184,3.07407225930411,"21/09/1801","23:28:00",7,66,1801-09-21 23:28:00 +"1555407","Atlantis","10",51.6100231290464,3.06001118563861,"21/09/1801","23:28:00",7,66,1801-09-21 23:28:00 +"1555408","Atlantis","10",51.547021713627,2.89482907915301,"22/09/1801"," 1:22:00",7,226,1801-09-22 01:22:00 +"1555409","Atlantis","10",51.5370821947339,2.87529989309702,"22/09/1801"," 5:12:00",5,96,1801-09-22 05:12:00 +"1555410","Atlantis","10",51.541101207267,2.85568756789807,"22/09/1801"," 7:08:00",7,46,1801-09-22 07:08:00 +"1555411","Atlantis","10",51.6432048825864,3.06129459817056,"22/09/1801"," 9:04:00",7,327,1801-09-22 09:04:00 +"1555412","Atlantis","10",51.64112977613,3.12847521154676,"22/09/1801","10:58:00",6,302,1801-09-22 10:58:00 +"1555413","Atlantis","10",51.6446539001246,3.14544970385451,"22/09/1801","12:54:00",6,183,1801-09-22 12:54:00 +"1555414","Atlantis","10",51.5335453653987,2.87610074411705,"22/09/1801","14:50:00",7,232,1801-09-22 14:50:00 +"1555415","Atlantis","10",51.5535119784218,2.84880464873929,"22/09/1801","20:34:00",7,308,1801-09-22 20:34:00 +"1555416","Atlantis","10",51.5474956538412,2.8758286859747,"22/09/1801","22:30:00",8,270,1801-09-22 22:30:00 +"1555417","Atlantis","10",51.5460690084323,2.89649960043747,"23/09/1801"," 2:20:00",7,62,1801-09-23 02:20:00 +"1555418","Atlantis","10",51.5514211131451,2.90268366619292,"23/09/1801"," 4:16:00",1,172,1801-09-23 04:16:00 +"1555419","Atlantis","10",51.5495581843243,2.85697844705079,"23/09/1801"," 6:10:00",6,282,1801-09-23 06:10:00 +"1555420","Atlantis","10",51.5253247259804,2.93190401869826,"23/09/1801"," 8:06:00",6,47,1801-09-23 08:06:00 +"1555421","Atlantis","10",51.517831816237,2.93616953798104,"23/09/1801"," 8:06:00",6,47,1801-09-23 08:06:00 +"1555422","Atlantis","10",51.5687327003279,3.03728956469335,"23/09/1801","10:02:00",5,235,1801-09-23 10:02:00 +"1555423","Atlantis","10",51.5480359526169,3.03731715998612,"23/09/1801","13:52:00",8,45,1801-09-23 13:52:00 +"1555424","Atlantis","10",51.459815652437,2.88228148337267,"23/09/1801","15:46:00",7,250,1801-09-23 15:46:00 +"1555425","Atlantis","10",51.3919987156806,2.60518348449562,"23/09/1801","17:42:00",7,249,1801-09-23 17:42:00 +"1555426","Atlantis","10",51.299196647421,2.3826450814344,"23/09/1801","19:36:00",6,209,1801-09-23 19:36:00 +"1555427","Atlantis","10",51.3144755225806,2.48406492707226,"23/09/1801","21:32:00",8,92,1801-09-23 21:32:00 +"1555428","Atlantis","10",51.4283873568,2.74755893425457,"23/09/1801","23:28:00",6,45,1801-09-23 23:28:00 +"1555429","Atlantis","10",51.5437486236542,2.96842689296883,"24/09/1801"," 1:22:00",6,16,1801-09-24 01:22:00 +"1555430","Atlantis","10",51.5686313771126,3.14876709777769,"24/09/1801"," 3:18:00",11,74,1801-09-24 03:18:00 +"1555431","Atlantis","10",51.4422775389431,3.60017206957005,"24/09/1801"," 5:14:00",5,306,1801-09-24 05:14:00 +"1555432","Atlantis","10",51.4456601491151,3.58971194659732,"24/09/1801"," 7:08:00",0,266,1801-09-24 07:08:00 +"1555433","Atlantis","10",51.4430409768759,3.58441896786355,"24/09/1801"," 9:04:00",0,266,1801-09-24 09:04:00 +"1555434","Atlantis","10",51.4415940989042,3.58721009844355,"24/09/1801","10:58:00",0,266,1801-09-24 10:58:00 +"1555435","Atlantis","10",51.4403935615346,3.58372482582368,"24/09/1801","12:54:00",0,266,1801-09-24 12:54:00 +"1555436","Atlantis","10",51.4464791745041,3.58346429321822,"24/09/1801","14:50:00",0,266,1801-09-24 14:50:00 +"1555437","Atlantis","10",51.44075881002,3.593833968224,"24/09/1801","16:44:00",0,266,1801-09-24 16:44:00 +"1558032","Atlantis","10",51.4416865218943,3.59898520351294,"26/09/1801","16:44:00",0,266,1801-09-26 16:44:00 +"1558033","Atlantis","10",51.444507561652,3.58986761561967,"26/09/1801","18:40:00",0,266,1801-09-26 18:40:00 +"1558034","Atlantis","10",51.4491718630446,3.59958758936636,"26/09/1801","20:34:00",0,266,1801-09-26 20:34:00 +"1558035","Atlantis","10",51.4469700975739,3.59976244255341,"26/09/1801","22:30:00",0,266,1801-09-26 22:30:00 +"1558036","Atlantis","10",51.4440934100258,3.58142909231503,"27/09/1801","00:26:00",0,266,1801-09-27 00:26:00 +"1558037","Atlantis","10",51.5682676910781,3.4050360801518,"27/09/1801"," 2:20:00",13,334,1801-09-27 02:20:00 +"1558038","Atlantis","10",51.9109818126173,3.0440084690284,"27/09/1801"," 4:16:00",13,334,1801-09-27 04:16:00 +"1558039","Atlantis","10",52.0014317679759,2.93187194291037,"27/09/1801"," 6:10:00",1,183,1801-09-27 06:10:00 +"1558040","Atlantis","10",52.0426699633361,2.91290088053234,"27/09/1801"," 8:06:00",3,98,1801-09-27 08:06:00 +"1558041","Atlantis","10",51.9973272840697,2.86476393927261,"27/09/1801","10:02:00",2,297,1801-09-27 10:02:00 +"1558042","Atlantis","10",51.9936305094389,2.95050260055531,"27/09/1801","11:56:00",6,100,1801-09-27 11:56:00 +"1558043","Atlantis","10",51.9916145632337,2.89220509236213,"27/09/1801","13:52:00",6,82,1801-09-27 13:52:00 +"1558044","Atlantis","10",52.0547769782222,2.74301726725139,"27/09/1801","15:46:00",6,304,1801-09-27 15:46:00 +"1558045","Atlantis","10",52.0614866272965,2.56911584057659,"27/09/1801","17:42:00",6,265,1801-09-27 17:42:00 +"1558046","Atlantis","10",52.1021421972322,2.45418863892369,"27/09/1801","19:38:00",5,167,1801-09-27 19:38:00 +"1558047","Atlantis","10",51.8635053773764,2.44897929977533,"27/09/1801","21:32:00",14,189,1801-09-27 21:32:00 +"1558048","Atlantis","10",51.5647645406644,2.26704707259778,"27/09/1801","23:28:00",6,227,1801-09-27 23:28:00 +"1558049","Atlantis","10",51.3949986578021,2.23858282744698,"28/09/1801"," 1:22:00",6,146,1801-09-28 01:22:00 +"1558050","Atlantis","10",51.4646490791421,2.33760085074324,"28/09/1801"," 3:18:00",7,49,1801-09-28 03:18:00 +"1558051","Atlantis","10",51.545809607177,2.52068897924666,"28/09/1801"," 5:14:00",6,215,1801-09-28 05:14:00 +"1558052","Atlantis","10",51.5283010063809,2.49850237113144,"28/09/1801"," 7:08:00",7,319,1801-09-28 07:08:00 +"1558053","Atlantis","10",51.5326686305585,2.51433237976302,"28/09/1801"," 9:04:00",3,91,1801-09-28 09:04:00 +"1558054","Atlantis","10",51.5231025944497,2.50100920245051,"28/09/1801","10:58:00",7,208,1801-09-28 10:58:00 +"1558055","Atlantis","10",51.3792545658685,2.29642233029846,"28/09/1801","12:54:00",5,201,1801-09-28 12:54:00 +"1558056","Atlantis","10",51.4779039047444,2.54954885991849,"28/09/1801","14:50:00",7,37,1801-09-28 14:50:00 +"1558057","Atlantis","10",51.5513259403594,2.63269043842983,"28/09/1801","16:44:00",9,60,1801-09-28 16:44:00 +"1558058","Atlantis","10",51.5519841404096,2.64188817282393,"28/09/1801","16:44:00",7,25,1801-09-28 16:44:00 +"1558059","Atlantis","10",51.5781261204942,2.67034391275607,"28/09/1801","18:40:00",6,212,1801-09-28 18:40:00 +"1558060","Atlantis","10",51.5803415251975,2.67682966098003,"28/09/1801","20:34:00",6,92,1801-09-28 20:34:00 +"1558061","Atlantis","10",51.5406331177237,2.57286354488134,"28/09/1801","22:30:00",7,198,1801-09-28 22:30:00 +"1558062","Atlantis","10",51.417292299855,2.46582766053081,"29/09/1801","00:26:00",4,39,1801-09-29 00:26:00 +"1558063","Atlantis","10",51.5286025949991,2.58255624916684,"29/09/1801"," 2:20:00",7,11,1801-09-29 02:20:00 +"1558064","Atlantis","10",51.5275327271684,2.57427951546479,"29/09/1801"," 4:16:00",5,194,1801-09-29 04:16:00 +"1558065","Atlantis","10",51.5050768146976,2.54326678625215,"29/09/1801"," 6:10:00",6,198,1801-09-29 06:10:00 +"1558066","Atlantis","10",51.5548413366494,2.57555594416708,"29/09/1801"," 8:06:00",6,18,1801-09-29 08:06:00 +"1558067","Atlantis","10",51.396479740486,2.53123941736296,"29/09/1801","10:02:00",6,34,1801-09-29 10:02:00 +"1558068","Atlantis","10",51.3657313061594,2.2747624752,"29/09/1801","13:52:00",12,301,1801-09-29 13:52:00 +"1558069","Atlantis","10",51.596758028449,1.91266153350473,"29/09/1801","15:46:00",7,16,1801-09-29 15:46:00 +"1558070","Atlantis","10",51.8183823845703,1.99119572775438,"29/09/1801","17:42:00",7,92,1801-09-29 17:42:00 +"1558071","Atlantis","10",51.7953515104749,2.14375905448478,"29/09/1801","19:38:00",12,105,1801-09-29 19:38:00 +"1558072","Atlantis","10",51.6509401507131,2.69030370800383,"29/09/1801","21:32:00",7,188,1801-09-29 21:32:00 +"1558073","Atlantis","10",51.4427872898132,2.58300456348806,"29/09/1801","23:28:00",2,233,1801-09-29 23:28:00 +"1558074","Atlantis","10",51.3148875146508,2.42021662644949,"30/09/1801"," 1:22:00",6,233,1801-09-30 01:22:00 +"1558075","Atlantis","10",51.3360548409061,2.51276356823835,"30/09/1801"," 3:18:00",7,34,1801-09-30 03:18:00 +"1558076","Atlantis","10",51.4402426631981,2.79793558369856,"30/09/1801"," 5:12:00",7,65,1801-09-30 05:12:00 +"1558077","Atlantis","10",51.5705081450548,3.08113315943349,"30/09/1801"," 7:08:00",7,7,1801-09-30 07:08:00 +"1558078","Atlantis","10",51.5700285940524,3.0875722193867,"30/09/1801"," 9:04:00",6,84,1801-09-30 09:04:00 +"1558079","Atlantis","10",51.5528322362537,3.03265564845782,"30/09/1801","10:28:00",5,49,1801-09-30 10:28:00 +"1558080","Atlantis","10",51.569113122283,3.09581196783483,"30/09/1801","10:58:00",7,195,1801-09-30 10:58:00 +"1558081","Atlantis","10",51.5629528832883,3.06528747584112,"30/09/1801","12:34:00",7,231,1801-09-30 12:34:00 +"1558082","Atlantis","10",51.5386679790528,3.01583193154726,"30/09/1801","12:54:00",6,229,1801-09-30 12:54:00 +"1558083","Atlantis","10",51.5566054825126,3.05772055265307,"30/09/1801","14:50:00",7,46,1801-09-30 14:50:00 +"1558084","Atlantis","10",51.5746130309128,3.06660910889227,"30/09/1801","15:30:00",6,267,1801-09-30 15:30:00 +"1558085","Atlantis","10",51.5547350416905,3.0379128654208,"30/09/1801","16:44:00",6,134,1801-09-30 16:44:00 +"1558086","Atlantis","10",51.5738008662565,2.91443522435706,"30/09/1801","18:40:00",7,312,1801-09-30 18:40:00 +"1558087","Atlantis","10",51.5816079751374,2.89790764666535,"30/09/1801","20:34:00",6,133,1801-09-30 20:34:00 +"1558088","Atlantis","10",51.5904997833855,2.95418330026325,"30/09/1801","22:30:00",5,42,1801-09-30 22:30:00 +"1558089","Atlantis","10",51.5388710420597,3.03839973628055,"01/10/1801","00:24:00",2,215,1801-10-01 00:24:00 +"1558090","Atlantis","10",51.5406176664173,3.07180915182456,"01/10/1801"," 2:20:00",11,101,1801-10-01 02:20:00 +"1558091","Atlantis","10",51.442142100336,3.58598931783345,"01/10/1801"," 4:16:00",0,296,1801-10-01 04:16:00 +"1558092","Atlantis","10",51.448800364316,3.59860111930314,"01/10/1801"," 6:10:00",0,93,1801-10-01 06:10:00 +"1558093","Atlantis","10",51.4408553106524,3.59381541414093,"01/10/1801"," 8:06:00",0,93,1801-10-01 08:06:00 +"1558094","Atlantis","10",51.4453299607523,3.58830646265205,"01/10/1801","10:02:00",0,93,1801-10-01 10:02:00 +"1558095","Atlantis","10",51.4477342711506,3.58414604367595,"01/10/1801","11:56:00",0,93,1801-10-01 11:56:00 +"1558096","Atlantis","10",51.4480933251628,3.58359541268926,"01/10/1801","13:52:00",0,93,1801-10-01 13:52:00 +"1558097","Atlantis","10",51.4471614910383,3.59122742063273,"01/10/1801","15:46:00",0,93,1801-10-01 15:46:00 +"1493719","Atlantis","1000",53.0434159252392,4.86039992010314,"06/09/1801"," 7:52:00",0,0,1801-09-06 07:52:00 +"1493720","Atlantis","1000",53.0391928949621,4.85573100547399,"06/09/1801"," 8:02:00",0,0,1801-09-06 08:02:00 +"1493721","Atlantis","1000",52.8786157848206,4.62145266381185,"06/09/1801"," 9:56:00",7,203,1801-09-06 09:56:00 +"1493722","Atlantis","1000",52.6651118033011,4.4838202519454,"06/09/1801","11:52:00",8,216,1801-09-06 11:52:00 +"1493723","Atlantis","1000",52.4562779938714,4.2572144849645,"06/09/1801","13:48:00",6,209,1801-09-06 13:48:00 +"1493724","Atlantis","1000",52.1225659727901,4.0045443233531,"06/09/1801","17:38:00",6,205,1801-09-06 17:38:00 +"1493725","Atlantis","1000",52.0587373466915,3.89711018315144,"06/09/1801","19:32:00",6,257,1801-09-06 19:32:00 +"1493726","Atlantis","1000",52.0879695585212,3.8076382340109,"06/09/1801","21:28:00",6,79,1801-09-06 21:28:00 +"1493727","Atlantis","1000",52.0943996549281,3.92925377265457,"06/09/1801","23:22:00",6,279,1801-09-06 23:22:00 +"1493728","Atlantis","1000",52.0927054601139,3.96246439004596,"07/09/1801"," 1:18:00",6,293,1801-09-07 01:18:00 +"1493729","Atlantis","1000",52.0253866966981,3.71335315156542,"07/09/1801"," 3:14:00",6,201,1801-09-07 03:14:00 +"1493730","Atlantis","1000",51.9032305564419,3.54204537094012,"07/09/1801"," 5:08:00",6,266,1801-09-07 05:08:00 +"1493731","Atlantis","1000",51.8173588924212,3.43941855607089,"07/09/1801"," 7:04:00",5,36,1801-09-07 07:04:00 +"1493732","Atlantis","1000",51.8248535044868,3.39852710428182,"07/09/1801"," 9:00:00",5,323,1801-09-07 09:00:00 +"1493733","Atlantis","1000",51.8922090479923,3.52017652205657,"07/09/1801","10:54:00",5,59,1801-09-07 10:54:00 +"1493734","Atlantis","1000",51.9213675553459,3.6420488694869,"07/09/1801","12:50:00",5,69,1801-09-07 12:50:00 +"1493735","Atlantis","1000",51.8913545836741,3.52167688786518,"07/09/1801","14:44:00",7,216,1801-09-07 14:44:00 +"1493736","Atlantis","1000",51.8116169474665,3.4032082766816,"07/09/1801","16:40:00",2,333,1801-09-07 16:40:00 +"1493737","Atlantis","1000",51.7950095808492,3.38747696138639,"07/09/1801","18:34:00",5,116,1801-09-07 18:34:00 +"1493738","Atlantis","1000",51.797432103775,3.38437063988485,"07/09/1801","18:34:00",5,116,1801-09-07 18:34:00 +"1493739","Atlantis","1000",51.7787881947774,3.35508410959877,"07/09/1801","20:30:00",2,237,1801-09-07 20:30:00 +"1493740","Atlantis","1000",51.8792570095169,3.46880157367978,"07/09/1801","22:26:00",4,64,1801-09-07 22:26:00 +"1493741","Atlantis","1000",51.9029700058745,3.58492405588459,"08/09/1801","00:20:00",5,64,1801-09-08 00:20:00 +"1493742","Atlantis","1000",51.9081234872513,3.57487422392703,"08/09/1801"," 2:16:00",7,212,1801-09-08 02:16:00 +"1493743","Atlantis","1000",51.8434572159438,3.43192138470989,"08/09/1801"," 4:10:00",5,162,1801-09-08 04:10:00 +"1493744","Atlantis","1000",51.8077329211943,3.4345757668959,"08/09/1801"," 6:06:00",5,31,1801-09-08 06:06:00 +"1493745","Atlantis","1000",51.8144763873336,3.36309118957631,"08/09/1801"," 8:02:00",0,328,1801-09-08 08:02:00 +"1493746","Atlantis","1000",51.8695243590521,3.45338490560465,"08/09/1801"," 9:56:00",4,68,1801-09-08 09:56:00 +"1493747","Atlantis","1000",51.9439359065038,3.64686058025341,"08/09/1801","11:52:00",5,42,1801-09-08 11:52:00 +"1493748","Atlantis","1000",52.0895222066161,3.77783132208604,"08/09/1801","13:46:00",5,24,1801-09-08 13:46:00 +"1493749","Atlantis","1000",52.074628931968,3.81902042715531,"08/09/1801","15:42:00",5,271,1801-09-08 15:42:00 +"1493750","Atlantis","1000",52.0505141853681,3.77911824222375,"08/09/1801","17:38:00",4,323,1801-09-08 17:38:00 +"1493751","Atlantis","1000",51.9478016214454,3.6316532344101,"08/09/1801","19:32:00",6,257,1801-09-08 19:32:00 +"1493752","Atlantis","1000",51.9255596830077,3.63964854777604,"08/09/1801","21:28:00",5,86,1801-09-08 21:28:00 +"1493753","Atlantis","1000",52.0790483078011,3.76505983921513,"08/09/1801","23:22:00",5,42,1801-09-08 23:22:00 +"1493754","Atlantis","1000",52.1189892186206,3.91242259835266,"09/09/1801"," 1:18:00",6,18,1801-09-09 01:18:00 +"1493755","Atlantis","1000",52.1977965749567,4.03307513936423,"09/09/1801"," 3:14:00",6,256,1801-09-09 03:14:00 +"1498316","Atlantis","1000",52.1377864535069,3.95042461542972,"09/09/1801"," 5:08:00",3,257,1801-09-09 05:08:00 +"1498317","Atlantis","1000",52.1039771344992,3.90317872717883,"09/09/1801"," 7:04:00",5,107,1801-09-09 07:04:00 +"1498318","Atlantis","1000",52.058072812485,3.72920788873173,"09/09/1801"," 9:00:00",6,215,1801-09-09 09:00:00 +"1498319","Atlantis","1000",51.9438128281957,3.57122923247237,"09/09/1801","10:54:00",5,232,1801-09-09 10:54:00 +"1498320","Atlantis","1000",51.8733214770723,3.60170960045699,"09/09/1801","12:50:00",6,53,1801-09-09 12:50:00 +"1498321","Atlantis","1000",51.9068747723,3.76527937569376,"09/09/1801","14:44:00",0,237,1801-09-09 14:44:00 +"1498322","Atlantis","1000",51.9079607464219,3.75440723025799,"09/09/1801","16:40:00",4,85,1801-09-09 16:40:00 +"1498323","Atlantis","1000",51.9230772269531,3.76959852884989,"09/09/1801","18:36:00",4,236,1801-09-09 18:36:00 +"1498324","Atlantis","1000",51.9160086156051,3.73847772618663,"09/09/1801","20:30:00",6,265,1801-09-09 20:30:00 +"1498325","Atlantis","1000",51.9140625757859,3.73183603986818,"09/09/1801","22:26:00",5,261,1801-09-09 22:26:00 +"1498326","Atlantis","1000",51.9216730084303,3.78951853413507,"10/09/1801","00:20:00",4,255,1801-09-10 00:20:00 +"1498327","Atlantis","1000",52.0070177473454,3.74435810361244,"10/09/1801"," 2:16:00",6,50,1801-09-10 02:16:00 +"1498328","Atlantis","1000",52.0857335051731,3.95902178882435,"10/09/1801"," 4:10:00",6,74,1801-09-10 04:10:00 +"1498329","Atlantis","1000",52.1020263903136,4.26419685279857,"10/09/1801"," 6:06:00",0,52,1801-09-10 06:06:00 +"1498330","Atlantis","1000",52.099411649066,4.26755675434135,"10/09/1801"," 8:02:00",0,52,1801-09-10 08:02:00 +"1498331","Atlantis","1000",52.097921293023,4.26781877661776,"10/09/1801"," 9:56:00",0,52,1801-09-10 09:56:00 +"1498332","Atlantis","1000",52.099881211027,4.26120797901601,"10/09/1801","11:52:00",0,52,1801-09-10 11:52:00 +"1498333","Atlantis","1000",52.105552203936,4.26040909489803,"10/09/1801","13:46:00",0,52,1801-09-10 13:46:00 +"1498334","Atlantis","1000",52.1015861303303,4.27405107150506,"10/09/1801","15:42:00",0,52,1801-09-10 15:42:00 +"1498335","Atlantis","1000",52.0990175321782,4.26158876291197,"10/09/1801","17:38:00",0,52,1801-09-10 17:38:00 +"1498381","Atlantis","1000",52.1057107336088,4.26755282586813,"14/09/1801","21:28:00",0,254,1801-09-14 21:28:00 +"1498382","Atlantis","1000",52.1023270196528,4.26463423678093,"14/09/1801","23:22:00",0,254,1801-09-14 23:22:00 +"1498383","Atlantis","1000",52.0965018301774,4.2636389182033,"15/09/1801"," 1:18:00",0,254,1801-09-15 01:18:00 +"1498384","Atlantis","1000",52.1042713696547,4.25777227747161,"15/09/1801"," 3:14:00",0,254,1801-09-15 03:14:00 +"1498385","Atlantis","1000",52.1048047531317,4.27222119367122,"15/09/1801"," 5:08:00",0,254,1801-09-15 05:08:00 +"1498386","Atlantis","1000",52.1008929254571,4.26494790281076,"15/09/1801"," 7:04:00",0,254,1801-09-15 07:04:00 +"1498387","Atlantis","1000",52.1950036195507,4.15780540184956,"15/09/1801"," 9:00:00",6,330,1801-09-15 09:00:00 +"1498388","Atlantis","1000",52.1849902982879,4.13210351084359,"15/09/1801","14:44:00",6,72,1801-09-15 14:44:00 +"1498389","Atlantis","1000",52.1666793614435,4.20295361393783,"15/09/1801","16:40:00",6,339,1801-09-15 16:40:00 +"1502939","Atlantis","1000",52.1961251398046,4.29312199095264,"15/09/1801","18:34:00",6,39,1801-09-15 18:34:00 +"1502940","Atlantis","1000",52.3458119147778,4.43052395474911,"15/09/1801","20:30:00",6,23,1801-09-15 20:30:00 +"1502941","Atlantis","1000",52.3384699962661,4.23194770198036,"15/09/1801","22:26:00",7,180,1801-09-15 22:26:00 +"1502942","Atlantis","1000",52.2380342839002,4.22860902754776,"16/09/1801","00:20:00",5,117,1801-09-16 00:20:00 +"1502943","Atlantis","1000",52.2273913278659,4.16403655192349,"16/09/1801"," 2:16:00",6,280,1801-09-16 02:16:00 +"1502944","Atlantis","1000",52.2230496009174,4.06598555628676,"16/09/1801"," 4:10:00",6,36,1801-09-16 04:10:00 +"1502945","Atlantis","1000",52.3393740825071,4.19296547806729,"16/09/1801"," 6:06:00",6,115,1801-09-16 06:06:00 +"1502946","Atlantis","1000",52.3513445997867,4.22717295426968,"16/09/1801"," 8:02:00",6,292,1801-09-16 08:02:00 +"1502947","Atlantis","1000",52.3566199783008,4.2469436198147,"16/09/1801"," 9:56:00",4,252,1801-09-16 09:56:00 +"1502948","Atlantis","1000",52.2465200651605,4.33544034792855,"16/09/1801","11:52:00",7,186,1801-09-16 11:52:00 +"1502949","Atlantis","1000",52.1827650199635,4.15517145046592,"16/09/1801","13:48:00",6,132,1801-09-16 13:48:00 +"1502950","Atlantis","1000",52.1740664670044,4.1620262981439,"16/09/1801","15:42:00",6,70,1801-09-16 15:42:00 +"1502951","Atlantis","1000",52.3210810701735,4.26800249421597,"16/09/1801","17:38:00",4,332,1801-09-16 17:38:00 +"1502952","Atlantis","1000",52.4992762543284,4.28171220722515,"16/09/1801","19:32:00",7,11,1801-09-16 19:32:00 +"1502953","Atlantis","1000",52.6744520382681,4.38678845097031,"16/09/1801","21:28:00",6,37,1801-09-16 21:28:00 +"1502954","Atlantis","1000",52.8052060831105,4.51107283907197,"16/09/1801","23:22:00",7,250,1801-09-16 23:22:00 +"1502955","Atlantis","1000",52.8370997888916,4.52895363173447,"17/09/1801"," 1:18:00",5,109,1801-09-17 01:18:00 +"1502956","Atlantis","1000",52.8020524877235,4.53773292665836,"17/09/1801"," 3:14:00",6,161,1801-09-17 03:14:00 +"1502957","Atlantis","1000",52.7444767662696,4.56012132533546,"17/09/1801"," 5:08:00",6,355,1801-09-17 05:08:00 +"1502958","Atlantis","1000",53.0344044417758,4.51360514394194,"17/09/1801"," 9:00:00",5,309,1801-09-17 09:00:00 +"1502959","Atlantis","1000",52.9342191733611,4.53843784791697,"17/09/1801","12:50:00",2,67,1801-09-17 12:50:00 +"1502960","Atlantis","1000",52.9199692168469,4.53248457246367,"17/09/1801","14:46:00",5,185,1801-09-17 14:46:00 +"1502961","Atlantis","1000",52.834716565839,4.46923180126306,"17/09/1801","16:40:00",5,101,1801-09-17 16:40:00 +"1502962","Atlantis","1000",52.8352378503825,4.50703666216135,"17/09/1801","18:36:00",8,64,1801-09-17 18:36:00 +"1502963","Atlantis","1000",52.8895322747892,4.50618051727489,"17/09/1801","22:26:00",5,293,1801-09-17 22:26:00 +"1502964","Atlantis","1000",52.9758086879603,4.36096535498649,"18/09/1801","00:20:00",5,350,1801-09-18 00:20:00 +"1502965","Atlantis","1000",52.8957018431895,4.45537037542183,"18/09/1801"," 2:16:00",5,105,1801-09-18 02:16:00 +"1502966","Atlantis","1000",52.8963877414763,4.55304866882786,"18/09/1801"," 4:10:00",5,101,1801-09-18 04:10:00 +"1502967","Atlantis","1000",52.9716838433202,4.78857516254764,"18/09/1801"," 6:06:00",7,127,1801-09-18 06:06:00 +"1502968","Atlantis","1000",52.9567094779066,4.78081506850477,"18/09/1801"," 8:02:00",0,0,1801-09-18 08:02:00 +"1502969","Atlantis","1000",53.0464929563478,4.85759623927623,"18/09/1801"," 9:56:00",0,22,1801-09-18 09:56:00 +"1502970","Atlantis","1000",53.0391089843512,4.85734033415653,"18/09/1801","11:52:00",0,22,1801-09-18 11:52:00 +"1502971","Atlantis","1000",53.0423311868235,4.86082503077201,"18/09/1801","13:46:00",0,22,1801-09-18 13:46:00 +"1502972","Atlantis","1000",53.0387893150835,4.85287608387973,"18/09/1801","15:42:00",0,22,1801-09-18 15:42:00 +"1502973","Atlantis","1000",53.0440184082552,4.84544364344422,"18/09/1801","17:38:00",0,22,1801-09-18 17:38:00 +"1507626","Atlantis","1000",53.0383256842936,4.85751508723572,"26/09/1801","11:52:00",0,22,1801-09-26 11:52:00 +"1507627","Atlantis","1000",53.0435960803526,4.85607188084256,"26/09/1801","13:48:00",0,22,1801-09-26 13:48:00 +"1507628","Atlantis","1000",53.0407864031307,4.86148977910262,"26/09/1801","15:42:00",0,22,1801-09-26 15:42:00 +"1507629","Atlantis","1000",53.0416237840611,4.85779454235081,"26/09/1801","17:38:00",0,22,1801-09-26 17:38:00 +"1507630","Atlantis","1000",53.0471266337736,4.86129902815074,"26/09/1801","19:32:00",0,22,1801-09-26 19:32:00 +"1507631","Atlantis","1000",53.0405591691723,4.85337611228786,"26/09/1801","21:28:00",0,22,1801-09-26 21:28:00 +"1507632","Atlantis","1000",53.0454804953258,4.68400068532675,"26/09/1801","23:24:00",8,286,1801-09-26 23:24:00 +"1507633","Atlantis","1000",52.9214232642087,4.53928504845407,"27/09/1801"," 1:18:00",6,204,1801-09-27 01:18:00 +"1507634","Atlantis","1000",52.9411627344843,4.4911656720927,"27/09/1801"," 3:14:00",5,298,1801-09-27 03:14:00 +"1507635","Atlantis","1000",52.9871179323704,4.523928632644,"27/09/1801"," 5:08:00",5,315,1801-09-27 05:08:00 +"1507636","Atlantis","1000",53.0175903242375,4.59168570462428,"27/09/1801"," 7:04:00",11,63,1801-09-27 07:04:00 +"1507637","Atlantis","1000",52.972059805667,4.50636399603263,"27/09/1801"," 8:58:00",3,319,1801-09-27 08:58:00 +"1507638","Atlantis","1000",53.0186845638072,4.56009708560072,"27/09/1801","10:54:00",5,149,1801-09-27 10:54:00 +"1507639","Atlantis","1000",52.8681100822063,4.65038123203255,"27/09/1801","12:50:00",1,118,1801-09-27 12:50:00 +"1507640","Atlantis","1000",52.7593794604233,4.58236632394884,"27/09/1801","14:44:00",5,287,1801-09-27 14:44:00 +"1507641","Atlantis","1000",52.8377421631296,4.43028642858192,"27/09/1801","16:40:00",6,83,1801-09-27 16:40:00 +"1507642","Atlantis","1000",52.8739825280649,4.52850564993545,"27/09/1801","18:36:00",6,3,1801-09-27 18:36:00 +"1507643","Atlantis","1000",53.0189731152924,4.53151862071268,"27/09/1801","20:30:00",5,81,1801-09-27 20:30:00 +"1507644","Atlantis","1000",53.0651295589269,4.41998828342091,"27/09/1801","22:26:00",5,286,1801-09-27 22:26:00 +"1507645","Atlantis","1000",53.003674019353,4.54597803249024,"28/09/1801","00:20:00",6,128,1801-09-28 00:20:00 +"1507646","Atlantis","1000",52.8373987062164,4.48244092096761,"28/09/1801"," 2:16:00",6,188,1801-09-28 02:16:00 +"1512204","Atlantis","1000",52.7211464689099,4.45581009457726,"28/09/1801"," 4:10:00",6,55,1801-09-28 04:10:00 +"1512205","Atlantis","1000",52.8848246160885,4.56567361652572,"28/09/1801"," 6:06:00",6,17,1801-09-28 06:06:00 +"1512206","Atlantis","1000",52.9685201412635,4.54201477331854,"28/09/1801"," 8:02:00",5,292,1801-09-28 08:02:00 +"1512207","Atlantis","1000",53.0413363377377,4.6127754547447,"28/09/1801"," 9:56:00",2,28,1801-09-28 09:56:00 +"1512208","Atlantis","1000",53.0015929468656,4.57043749057129,"28/09/1801","11:52:00",4,269,1801-09-28 11:52:00 +"1512209","Atlantis","1000",52.9699303423516,4.38927855264861,"28/09/1801","13:48:00",6,263,1801-09-28 13:48:00 +"1512210","Atlantis","1000",53.0942929543839,4.29571476837154,"28/09/1801","15:42:00",6,325,1801-09-28 15:42:00 +"1512211","Atlantis","1000",53.2568992878282,4.22876921610255,"28/09/1801","17:38:00",7,12,1801-09-28 17:38:00 +"1512212","Atlantis","1000",53.4367072860659,4.28098407425545,"28/09/1801","19:32:00",6,358,1801-09-28 19:32:00 +"1512213","Atlantis","1000",53.4859683703231,4.25668919563666,"28/09/1801","21:28:00",5,180,1801-09-28 21:28:00 +"1512214","Atlantis","1000",53.4882732125446,4.27515190379415,"28/09/1801","23:22:00",6,110,1801-09-28 23:22:00 +"1512215","Atlantis","1000",53.4636889837482,4.52624924775306,"29/09/1801"," 1:18:00",5,58,1801-09-29 01:18:00 +"1512216","Atlantis","1000",53.4951235034154,4.30082263695169,"29/09/1801"," 3:14:00",5,282,1801-09-29 03:14:00 +"1512217","Atlantis","1000",53.4089845649297,4.4895406128522,"29/09/1801"," 5:08:00",6,141,1801-09-29 05:08:00 +"1512218","Atlantis","1000",53.4167837433522,4.44760875632614,"29/09/1801"," 7:04:00",5,337,1801-09-29 07:04:00 +"1512219","Atlantis","1000",53.4141744814115,4.43579513007589,"29/09/1801"," 8:58:00",7,111,1801-09-29 08:58:00 +"1512220","Atlantis","1000",53.3896926419199,4.40086957068741,"29/09/1801","12:50:00",4,147,1801-09-29 12:50:00 +"1512221","Atlantis","1000",53.382714761775,4.49920261679031,"29/09/1801","14:44:00",0,164,1801-09-29 14:44:00 +"1512222","Atlantis","1000",53.3952453846992,4.51067131389771,"29/09/1801","16:40:00",5,112,1801-09-29 16:40:00 +"1512223","Atlantis","1000",53.4176659024577,4.507446687866,"29/09/1801","18:36:00",5,289,1801-09-29 18:36:00 +"1512224","Atlantis","1000",53.3801557621583,4.53723492761236,"29/09/1801","20:30:00",6,91,1801-09-29 20:30:00 +"1512225","Atlantis","1000",53.4352834003186,4.64812511865422,"29/09/1801","22:26:00",6,305,1801-09-29 22:26:00 +"1512226","Atlantis","1000",53.3494218198801,4.46770379958768,"30/09/1801","00:20:00",1,282,1801-09-30 00:20:00 +"1512227","Atlantis","1000",53.3424762216341,4.34697645178996,"30/09/1801"," 2:16:00",5,76,1801-09-30 02:16:00 +"1512228","Atlantis","1000",53.3370280859522,4.42569616551325,"30/09/1801"," 4:10:00",5,161,1801-09-30 04:10:00 +"1512229","Atlantis","1000",53.3558751911884,4.47312990063894,"30/09/1801"," 6:06:00",6,28,1801-09-30 06:06:00 +"1512230","Atlantis","1000",53.40505229514,4.31221120379679,"30/09/1801"," 8:02:00",6,138,1801-09-30 08:02:00 +"1512231","Atlantis","1000",53.3898518887577,4.33386715704575,"30/09/1801"," 9:56:00",5,280,1801-09-30 09:56:00 +"1512232","Atlantis","1000",53.3874105561376,4.30660998167191,"30/09/1801","11:52:00",6,281,1801-09-30 11:52:00 +"1512233","Atlantis","1000",53.3878246128513,4.25705507229734,"30/09/1801","13:46:00",6,133,1801-09-30 13:46:00 +"1512234","Atlantis","1000",53.3582406851947,4.25134513336979,"30/09/1801","15:42:00",6,175,1801-09-30 15:42:00 +"1512235","Atlantis","1000",53.4051861366234,4.2894739327291,"30/09/1801","17:38:00",5,180,1801-09-30 17:38:00 +"1512236","Atlantis","1000",53.3936309569199,4.26617501014751,"30/09/1801","19:32:00",6,356,1801-09-30 19:32:00 +"1512237","Atlantis","1000",53.355841866516,4.39583778709732,"30/09/1801","21:28:00",5,159,1801-09-30 21:28:00 +"1512238","Atlantis","1000",53.2234248305834,4.38129454640485,"30/09/1801","23:24:00",6,181,1801-09-30 23:24:00 +"1512239","Atlantis","1000",53.0655651656962,4.49305760519858,"01/10/1801"," 1:18:00",6,197,1801-10-01 01:18:00 +"1512240","Atlantis","1000",52.92363798668,4.5444898406826,"01/10/1801"," 3:14:00",6,192,1801-10-01 03:14:00 +"1512241","Atlantis","1000",52.8624327646787,4.49109413930867,"01/10/1801"," 5:08:00",6,315,1801-10-01 05:08:00 +"1512242","Atlantis","1000",52.917895494231,4.66352581830509,"01/10/1801"," 7:04:00",10,35,1801-10-01 07:04:00 +"1512243","Atlantis","1000",52.9525603733738,4.77155911312625,"01/10/1801"," 8:58:00",0,164,1801-10-01 08:58:00 +"1512244","Atlantis","1000",53.0458838200402,4.84651307640691,"01/10/1801","10:54:00",0,21,1801-10-01 10:54:00 +"1512245","Atlantis","1000",53.0423886302924,4.85602574569732,"01/10/1801","12:50:00",0,21,1801-10-01 12:50:00 +"1512246","Atlantis","1000",53.0422707421188,4.85222774203494,"01/10/1801","14:44:00",0,21,1801-10-01 14:44:00 +"1512247","Atlantis","1000",53.0387854943001,4.86357425378263,"01/10/1801","16:40:00",0,21,1801-10-01 16:40:00 +"1512248","Atlantis","1000",53.0384856555746,4.85492216722667,"01/10/1801","18:34:00",0,21,1801-10-01 18:34:00 +"755434","Atlantis","1004",53.0410637151897,4.858329753072,"04/05/1800","22:22:00",0,36,1800-05-04 22:22:00 +"755435","Atlantis","1004",53.0398865111154,4.85689350460842,"04/05/1800","23:54:00",0,36,1800-05-04 23:54:00 +"755436","Atlantis","1004",53.0438210933241,4.8530598966144,"05/05/1800"," 2:06:00",0,36,1800-05-05 02:06:00 +"755437","Atlantis","1004",53.0425317819538,4.85701781604625,"05/05/1800"," 4:04:00",0,36,1800-05-05 04:04:00 +"755438","Atlantis","1004",53.0468528502602,4.85965077394806,"05/05/1800"," 5:34:00",0,36,1800-05-05 05:34:00 +"755439","Atlantis","1004",52.895218875133,4.57177000698354,"05/05/1800"," 9:30:00",13,253,1800-05-05 09:30:00 +"755440","Atlantis","1004",52.8443452505716,4.32842034869175,"05/05/1800","11:16:00",7,36,1800-05-05 11:16:00 +"755441","Atlantis","1004",52.8427088121311,4.27484300872404,"05/05/1800","13:44:00",7,103,1800-05-05 13:44:00 +"755442","Atlantis","1004",52.8156396426102,4.28638469165098,"05/05/1800","15:08:00",7,198,1800-05-05 15:08:00 +"755443","Atlantis","1004",52.9883903364171,4.28869952339493,"05/05/1800","17:36:00",7,7,1800-05-05 17:36:00 +"755444","Atlantis","1004",52.9975110820504,4.22593754480313,"05/05/1800","19:04:00",6,196,1800-05-05 19:04:00 +"755445","Atlantis","1004",53.0120883995886,4.18397610693797,"05/05/1800","21:20:00",7,160,1800-05-05 21:20:00 +"755446","Atlantis","1004",52.8212061220631,4.1863530101832,"05/05/1800","23:12:00",7,200,1800-05-05 23:12:00 +"755447","Atlantis","1004",52.7768871428445,4.28019045575429,"06/05/1800","00:46:00",7,143,1800-05-06 00:46:00 +"755448","Atlantis","1004",52.6479988184483,4.25120409695804,"06/05/1800"," 3:08:00",7,20,1800-05-06 03:08:00 +"755449","Atlantis","1004",52.7312412026022,4.27164341187105,"06/05/1800"," 4:46:00",7,184,1800-05-06 04:46:00 +"755450","Atlantis","1004",52.7746488657403,4.23060119117331,"06/05/1800"," 6:36:00",7,276,1800-05-06 06:36:00 +"755451","Atlantis","1004",52.8115275521972,4.20150245377421,"06/05/1800"," 8:42:00",7,304,1800-05-06 08:42:00 +"755452","Atlantis","1004",52.9582336360016,4.13177466383577,"06/05/1800","10:32:00",1,33,1800-05-06 10:32:00 +"755453","Atlantis","1004",52.8105765686468,4.02608461197093,"06/05/1800","12:36:00",6,185,1800-05-06 12:36:00 +"755454","Atlantis","1004",52.6277495644484,3.92510407305416,"06/05/1800","14:20:00",7,195,1800-05-06 14:20:00 +"755455","Atlantis","1004",52.6563245558548,3.9138392011011,"06/05/1800","16:08:00",7,283,1800-05-06 16:08:00 +"755456","Atlantis","1004",52.6820222244021,3.96188473174162,"06/05/1800","18:14:00",7,24,1800-05-06 18:14:00 +"755457","Atlantis","1004",52.7137377939639,3.97940136934072,"06/05/1800","20:18:00",7,248,1800-05-06 20:18:00 +"755458","Atlantis","1004",52.6831122465655,3.94130963684246,"06/05/1800","21:52:00",7,21,1800-05-06 21:52:00 +"755459","Atlantis","1004",52.5836684614932,3.81500060566608,"07/05/1800","00:18:00",7,272,1800-05-07 00:18:00 +"755460","Atlantis","1004",52.6078598592002,3.79763066495769,"07/05/1800"," 1:58:00",7,3,1800-05-07 01:58:00 +"755461","Atlantis","1004",52.6861749686715,3.94431631326955,"07/05/1800"," 3:42:00",8,235,1800-05-07 03:42:00 +"755462","Atlantis","1004",52.781947661784,4.0400970756039,"07/05/1800"," 5:36:00",5,41,1800-05-07 05:36:00 +"755463","Atlantis","1004",52.8244807097595,4.0866260997504,"07/05/1800"," 7:46:00",7,331,1800-05-07 07:46:00 +"755464","Atlantis","1004",52.8752356817019,4.05530407363828,"07/05/1800"," 9:34:00",6,235,1800-05-07 09:34:00 +"755465","Atlantis","1004",52.7391884190724,4.01452104163077,"07/05/1800","11:16:00",7,186,1800-05-07 11:16:00 +"755466","Atlantis","1004",52.6217233637175,3.97294506056048,"07/05/1800","13:34:00",8,207,1800-05-07 13:34:00 +"755467","Atlantis","1004",52.572891259009,3.90145046227984,"07/05/1800","15:32:00",7,20,1800-05-07 15:32:00 +"759373","Atlantis","1004",52.694721649827,3.93952235780098,"07/05/1800","17:22:00",0,315,1800-05-07 17:22:00 +"759374","Atlantis","1004",52.7403297130312,3.86636112119071,"08/05/1800"," 1:12:00",7,330,1800-05-08 01:12:00 +"759375","Atlantis","1004",52.7663727129912,3.9763517176602,"08/05/1800"," 3:06:00",7,234,1800-05-08 03:06:00 +"759376","Atlantis","1004",52.7820742196948,3.96445956499688,"08/05/1800"," 4:50:00",7,92,1800-05-08 04:50:00 +"759377","Atlantis","1004",52.7694344382365,3.91430960608367,"08/05/1800"," 6:32:00",7,98,1800-05-08 06:32:00 +"759378","Atlantis","1004",52.765488364893,3.99230497984216,"08/05/1800"," 8:58:00",8,66,1800-05-08 08:58:00 +"759379","Atlantis","1004",52.7545567548112,3.90839323052764,"08/05/1800","10:48:00",7,291,1800-05-08 10:48:00 +"759380","Atlantis","1004",52.8990048942389,3.90936921841651,"08/05/1800","12:28:00",7,129,1800-05-08 12:28:00 +"759381","Atlantis","1004",52.9940440721009,3.98881037372164,"08/05/1800","14:10:00",8,188,1800-05-08 14:10:00 +"759382","Atlantis","1004",52.813348988255,3.93535076831654,"08/05/1800","16:20:00",7,347,1800-05-08 16:20:00 +"759383","Atlantis","1004",52.8572157871951,3.93399148480594,"08/05/1800","20:00:00",7,209,1800-05-08 20:00:00 +"759384","Atlantis","1004",52.791672152814,3.89442145360261,"08/05/1800","22:16:00",7,2,1800-05-08 22:16:00 +"759385","Atlantis","1004",52.737590528958,3.88779468375724,"09/05/1800","00:10:00",7,188,1800-05-09 00:10:00 +"759386","Atlantis","1004",52.7510031365757,3.88584089571703,"09/05/1800"," 2:00:00",9,179,1800-05-09 02:00:00 +"759387","Atlantis","1004",52.7731042947639,3.91580103338696,"09/05/1800"," 3:34:00",7,182,1800-05-09 03:34:00 +"759388","Atlantis","1004",52.8288837654921,4.31737401986588,"09/05/1800"," 5:46:00",13,69,1800-05-09 05:46:00 +"759389","Atlantis","1004",52.9568596659838,4.76728857636266,"09/05/1800"," 7:44:00",0,201,1800-05-09 07:44:00 +"759390","Atlantis","1004",52.9570782294339,4.773146422741,"09/05/1800"," 9:20:00",0,201,1800-05-09 09:20:00 +"759391","Atlantis","1004",53.0433751153802,4.84980009037722,"09/05/1800","11:22:00",0,30,1800-05-09 11:22:00 +"759392","Atlantis","1004",53.0380895836716,4.85131246694084,"09/05/1800","13:44:00",0,30,1800-05-09 13:44:00 +"759393","Atlantis","1004",53.0465337839322,4.85769998701475,"09/05/1800","15:18:00",1,220,1800-05-09 15:18:00 +"759394","Atlantis","1004",53.0417720127171,4.84966114646476,"09/05/1800","17:26:00",0,220,1800-05-09 17:26:00 +"759395","Atlantis","1004",53.0473312837742,4.8458476259904,"09/05/1800","19:10:00",0,220,1800-05-09 19:10:00 +"1479869","Atlantis","1004",53.0392344388706,4.86036410922836,"05/09/1801","21:14:00",0,7,1801-09-05 21:14:00 +"1479870","Atlantis","1004",53.0423659209642,4.84364130577725,"05/09/1801","23:10:00",0,7,1801-09-05 23:10:00 +"1479871","Atlantis","1004",53.0457082244791,4.8539554268755,"06/09/1801"," 1:04:00",0,7,1801-09-06 01:04:00 +"1479872","Atlantis","1004",53.0434592928099,4.84513566746563,"06/09/1801"," 3:00:00",0,7,1801-09-06 03:00:00 +"1479873","Atlantis","1004",53.0451938302992,4.86182668678183,"06/09/1801"," 4:54:00",0,7,1801-09-06 04:54:00 +"1479874","Atlantis","1004",53.0445346946493,4.8507873154683,"06/09/1801"," 6:50:00",0,7,1801-09-06 06:50:00 +"1479875","Atlantis","1004",53.002573111725,4.703396786822,"06/09/1801"," 8:46:00",13,344,1801-09-06 08:46:00 +"1479876","Atlantis","1004",53.4089336687778,5.0119606510736,"06/09/1801","10:40:00",14,21,1801-09-06 10:40:00 +"1479877","Atlantis","1004",53.801261431986,5.27294915364497,"06/09/1801","12:36:00",13,19,1801-09-06 12:36:00 +"1479878","Atlantis","1004",54.1545395930749,5.46871859058831,"06/09/1801","14:30:00",0,146,1801-09-06 14:30:00 +"1479879","Atlantis","1004",54.3405738733252,5.50832696464471,"06/09/1801","16:26:00",2,352,1801-09-06 16:26:00 +"1479880","Atlantis","1004",54.4102298285351,5.82043598877732,"06/09/1801","18:22:00",7,66,1801-09-06 18:22:00 +"1479881","Atlantis","1004",54.426228788998,6.12755554681644,"06/09/1801","20:16:00",7,94,1801-09-06 20:16:00 +"1479882","Atlantis","1004",54.4099782569995,6.43957572595962,"06/09/1801","22:12:00",7,105,1801-09-06 22:12:00 +"1479883","Atlantis","1004",54.4076076412904,6.76971680895705,"07/09/1801","00: 6:00",7,83,1801-09-07 00:06:00 +"1479884","Atlantis","1004",54.3241370450603,7.07410840429273,"07/09/1801"," 2:02:00",8,139,1801-09-07 02:02:00 +"1479885","Atlantis","1004",54.3574650440938,7.05944719422143,"07/09/1801"," 3:58:00",7,339,1801-09-07 03:58:00 +"1479886","Atlantis","1004",54.4601323607746,6.82051229448803,"07/09/1801"," 5:52:00",8,67,1801-09-07 05:52:00 +"1479887","Atlantis","1004",54.4677537649819,6.81464160673134,"07/09/1801"," 7:48:00",6,280,1801-09-07 07:48:00 +"1479888","Atlantis","1004",54.4646518317168,6.42561809108965,"07/09/1801"," 9:42:00",4,50,1801-09-07 09:42:00 +"1479889","Atlantis","1004",54.6323866622695,6.28308853371627,"07/09/1801","11:38:00",4,347,1801-09-07 11:38:00 +"1479890","Atlantis","1004",54.7407147764154,6.20642765259836,"07/09/1801","13:34:00",7,271,1801-09-07 13:34:00 +"1479891","Atlantis","1004",54.4458198879589,6.34269926614314,"07/09/1801","17:24:00",7,175,1801-09-07 17:24:00 +"1484459","Atlantis","1004",54.3042274570023,6.28869009834714,"07/09/1801","19:18:00",7,171,1801-09-07 19:18:00 +"1484460","Atlantis","1004",54.2665358810923,6.42294426100794,"07/09/1801","21:14:00",7,104,1801-09-07 21:14:00 +"1484461","Atlantis","1004",54.3018953923653,6.37204595932365,"07/09/1801","23:10:00",7,340,1801-09-07 23:10:00 +"1484462","Atlantis","1004",54.4089804479578,6.33021399284154,"08/09/1801"," 1:04:00",7,160,1801-09-08 01:04:00 +"1484463","Atlantis","1004",54.2849679120462,6.36460551559832,"08/09/1801"," 3:00:00",7,143,1801-09-08 03:00:00 +"1484464","Atlantis","1004",54.4193463754496,6.28005679230485,"08/09/1801"," 4:56:00",1,169,1801-09-08 04:56:00 +"1484465","Atlantis","1004",54.469061608905,6.29027570619434,"08/09/1801"," 6:50:00",7,45,1801-09-08 06:50:00 +"1484466","Atlantis","1004",54.4791815830991,6.41702589099016,"08/09/1801"," 8:46:00",7,269,1801-09-08 08:46:00 +"1484467","Atlantis","1004",54.4944852960217,6.42948349237815,"08/09/1801","10:40:00",7,265,1801-09-08 10:40:00 +"1484468","Atlantis","1004",54.4989037375459,6.40030610878114,"08/09/1801","12:36:00",7,256,1801-09-08 12:36:00 +"1484469","Atlantis","1004",54.4632852229797,6.37143562685605,"08/09/1801","14:32:00",7,178,1801-09-08 14:32:00 +"1484470","Atlantis","1004",54.5497082880992,6.34017423263658,"08/09/1801","16:26:00",1,196,1801-09-08 16:26:00 +"1484471","Atlantis","1004",54.5108095879457,6.45603880997002,"08/09/1801","18:22:00",5,76,1801-09-08 18:22:00 +"1484472","Atlantis","1004",54.5897526759044,6.32924884794839,"08/09/1801","22:12:00",7,158,1801-09-08 22:12:00 +"1484473","Atlantis","1004",54.6439446739308,6.2994521883931,"09/09/1801","00: 6:00",7,340,1801-09-09 00:06:00 +"1484474","Atlantis","1004",54.5671804151614,6.35279353771917,"09/09/1801"," 3:58:00",7,316,1801-09-09 03:58:00 +"1484475","Atlantis","1004",54.5410447421949,6.37194776406698,"09/09/1801"," 5:52:00",7,321,1801-09-09 05:52:00 +"1484476","Atlantis","1004",54.5561973582567,6.35854177137557,"09/09/1801"," 7:48:00",6,144,1801-09-09 07:48:00 +"1484477","Atlantis","1004",54.5479606093392,6.35978876614291,"09/09/1801"," 9:44:00",5,129,1801-09-09 09:44:00 +"1484478","Atlantis","1004",54.5591612695865,6.32393997839186,"09/09/1801","11:38:00",7,125,1801-09-09 11:38:00 +"1484479","Atlantis","1004",54.5631873902399,6.31274870878365,"09/09/1801","13:34:00",7,127,1801-09-09 13:34:00 +"1484480","Atlantis","1004",54.5400646583047,5.88101539776568,"09/09/1801","19:20:00",8,210,1801-09-09 19:20:00 +"1484481","Atlantis","1004",54.3736211480629,5.70712033914495,"09/09/1801","21:14:00",7,205,1801-09-09 21:14:00 +"1484482","Atlantis","1004",54.1810130257835,5.55025073332246,"09/09/1801","23:10:00",1,287,1801-09-09 23:10:00 +"1484483","Atlantis","1004",54.050464284583,5.30522555922903,"10/09/1801"," 1:04:00",3,56,1801-09-10 01:04:00 +"1484484","Atlantis","1004",53.6680950363739,5.06148932672665,"10/09/1801"," 3:00:00",12,201,1801-09-10 03:00:00 +"1484485","Atlantis","1004",52.9694738769522,4.7607462142054,"10/09/1801"," 6:50:00",12,98,1801-09-10 06:50:00 +"1484486","Atlantis","1004",52.9548190539056,4.77466459030006,"10/09/1801"," 8:46:00",0,205,1801-09-10 08:46:00 +"1484487","Atlantis","1004",53.0374713666416,4.85216495376732,"10/09/1801","14:32:00",0,38,1801-09-10 14:32:00 +"1484488","Atlantis","1004",53.0447675585635,4.84586072135437,"10/09/1801","16:26:00",0,38,1801-09-10 16:26:00 +"1484526","Atlantis","1004",53.0413606886338,4.84757613166887,"14/09/1801"," 3:00:00",0,38,1801-09-14 03:00:00 +"1484527","Atlantis","1004",53.0450951574049,4.84715148185566,"14/09/1801"," 4:54:00",0,38,1801-09-14 04:54:00 +"1484528","Atlantis","1004",53.0406464351206,4.85143703424558,"14/09/1801"," 6:50:00",0,38,1801-09-14 06:50:00 +"1484529","Atlantis","1004",53.0430446803528,4.84530030579027,"14/09/1801"," 8:46:00",0,38,1801-09-14 08:46:00 +"1484530","Atlantis","1004",53.037455268668,4.85720497497078,"14/09/1801","12:36:00",0,38,1801-09-14 12:36:00 +"1484531","Atlantis","1004",53.023373687082,4.843852100241,"14/09/1801","14:30:00",13,225,1801-09-14 14:30:00 +"1484532","Atlantis","1004",53.3137383029661,4.84984935967438,"14/09/1801","16:26:00",14,29,1801-09-14 16:26:00 +"1489103","Atlantis","1004",53.684532109038,5.25305869249534,"14/09/1801","18:22:00",14,53,1801-09-14 18:22:00 +"1489104","Atlantis","1004",53.8541171006858,5.75524859377555,"14/09/1801","20:16:00",8,92,1801-09-14 20:16:00 +"1489105","Atlantis","1004",53.8501951747988,5.84582387269195,"14/09/1801","22:12:00",8,76,1801-09-14 22:12:00 +"1489106","Atlantis","1004",53.8670559113249,5.73439797494188,"15/09/1801","00: 6:00",2,84,1801-09-15 00:06:00 +"1489107","Atlantis","1004",53.8685538852597,5.77221434626263,"15/09/1801"," 2:02:00",7,260,1801-09-15 02:02:00 +"1489108","Atlantis","1004",53.8725501399008,5.83652434122376,"15/09/1801"," 3:58:00",7,272,1801-09-15 03:58:00 +"1489109","Atlantis","1004",53.8963125391765,5.94338676827121,"15/09/1801"," 5:52:00",7,66,1801-09-15 05:52:00 +"1489110","Atlantis","1004",53.879937529047,5.90622463855892,"15/09/1801"," 7:48:00",7,53,1801-09-15 07:48:00 +"1489111","Atlantis","1004",53.8693657158637,5.69412082180474,"15/09/1801"," 9:42:00",9,43,1801-09-15 09:42:00 +"1489112","Atlantis","1004",53.8584036023691,5.73459266796056,"15/09/1801","11:38:00",7,273,1801-09-15 11:38:00 +"1489113","Atlantis","1004",53.8682168937246,5.83647920089494,"15/09/1801","13:34:00",8,272,1801-09-15 13:34:00 +"1489114","Atlantis","1004",53.8650847992883,5.77495785683393,"15/09/1801","15:28:00",8,71,1801-09-15 15:28:00 +"1489115","Atlantis","1004",53.8627589943446,5.54314264584333,"15/09/1801","17:24:00",5,96,1801-09-15 17:24:00 +"1489116","Atlantis","1004",53.8699970897539,5.76539078393113,"15/09/1801","19:20:00",7,265,1801-09-15 19:20:00 +"1489117","Atlantis","1004",53.8652906453027,5.71726682690717,"15/09/1801","21:14:00",7,265,1801-09-15 21:14:00 +"1489118","Atlantis","1004",53.8797924722652,5.76718238819018,"15/09/1801","23:10:00",7,76,1801-09-15 23:10:00 +"1489119","Atlantis","1004",53.8728242186573,5.65034316006303,"16/09/1801"," 1:04:00",0,163,1801-09-16 01:04:00 +"1489120","Atlantis","1004",53.8768178287898,5.67966109550744,"16/09/1801"," 3:00:00",7,276,1801-09-16 03:00:00 +"1489121","Atlantis","1004",53.8767119087656,5.84437740060501,"16/09/1801"," 4:54:00",9,216,1801-09-16 04:54:00 +"1489122","Atlantis","1004",53.8571040251367,5.80541348170675,"16/09/1801"," 6:50:00",7,94,1801-09-16 06:50:00 +"1489123","Atlantis","1004",53.86117398979,5.6849948577825,"16/09/1801"," 8:46:00",1,111,1801-09-16 08:46:00 +"1489124","Atlantis","1004",53.8847196335993,5.75747202584986,"16/09/1801","10:40:00",7,277,1801-09-16 10:40:00 +"1489125","Atlantis","1004",53.8865175444987,5.82370227187965,"16/09/1801","12:36:00",7,297,1801-09-16 12:36:00 +"1489126","Atlantis","1004",53.8889822226651,5.8447759864293,"16/09/1801","14:30:00",7,132,1801-09-16 14:30:00 +"1489127","Atlantis","1004",53.880980653719,5.74309996145032,"16/09/1801","16:26:00",1,271,1801-09-16 16:26:00 +"1489128","Atlantis","1004",53.8731739558443,5.75912413315661,"16/09/1801","22:12:00",7,271,1801-09-16 22:12:00 +"1489129","Atlantis","1004",53.8546077947901,5.560710162987,"17/09/1801"," 2:02:00",3,308,1801-09-17 02:02:00 +"1489130","Atlantis","1004",53.7225079750507,5.51729022800457,"17/09/1801"," 3:58:00",13,194,1801-09-17 03:58:00 +"1489131","Atlantis","1004",53.4127458363031,5.04133749642409,"17/09/1801"," 5:52:00",13,232,1801-09-17 05:52:00 +"1489132","Atlantis","1004",53.102333352969,4.69778899929393,"17/09/1801"," 7:48:00",12,196,1801-09-17 07:48:00 +"1489133","Atlantis","1004",52.9545030079903,4.78054228164069,"17/09/1801"," 9:44:00",0,186,1801-09-17 09:44:00 +"1489134","Atlantis","1004",52.955386594255,4.78120440690126,"17/09/1801","11:38:00",0,186,1801-09-17 11:38:00 +"1489135","Atlantis","1004",52.9562979692249,4.78219287473056,"17/09/1801","13:34:00",0,186,1801-09-17 13:34:00 +"1489136","Atlantis","1004",53.0432727451702,4.85335730643943,"17/09/1801","15:28:00",5,288,1801-09-17 15:28:00 +"1489137","Atlantis","1004",53.0415816103467,4.84362992135249,"17/09/1801","17:24:00",0,266,1801-09-17 17:24:00 +"1489138","Atlantis","1004",53.0391540483581,4.84586369207036,"17/09/1801","19:20:00",0,266,1801-09-17 19:20:00 +"1489159","Atlantis","1004",53.041383995126,4.84326378515829,"19/09/1801","21:14:00",0,266,1801-09-19 21:14:00 +"1489160","Atlantis","1004",53.0392122051036,4.85009649136476,"19/09/1801","23:10:00",0,266,1801-09-19 23:10:00 +"1489161","Atlantis","1004",53.0415948440074,4.85166649456974,"20/09/1801"," 1:04:00",0,266,1801-09-20 01:04:00 +"1489162","Atlantis","1004",53.0358263468402,4.85625621794164,"20/09/1801"," 3:00:00",0,266,1801-09-20 03:00:00 +"1489163","Atlantis","1004",53.0370300040217,4.85940674014296,"20/09/1801"," 4:56:00",0,266,1801-09-20 04:56:00 +"1489164","Atlantis","1004",52.9802406552658,4.7601873349091,"20/09/1801"," 8:46:00",9,237,1801-09-20 08:46:00 +"1489165","Atlantis","1004",53.353141011151,4.91828634655382,"20/09/1801","10:40:00",14,39,1801-09-20 10:40:00 +"1489166","Atlantis","1004",53.7139946421618,5.37867092360742,"20/09/1801","12:36:00",14,39,1801-09-20 12:36:00 +"1489167","Atlantis","1004",53.8703608419439,5.76477424832154,"20/09/1801","14:32:00",7,92,1801-09-20 14:32:00 +"1489168","Atlantis","1004",53.9073997766292,5.72150444294047,"20/09/1801","16:26:00",7,256,1801-09-20 16:26:00 +"1489169","Atlantis","1004",53.904886003525,5.57220428710524,"20/09/1801","18:22:00",7,99,1801-09-20 18:22:00 +"1489170","Atlantis","1004",53.8877062355555,5.48213320673164,"20/09/1801","20:18:00",5,80,1801-09-20 20:18:00 +"1489171","Atlantis","1004",53.8608302167985,5.40583233108465,"20/09/1801","22:12:00",6,279,1801-09-20 22:12:00 +"1489172","Atlantis","1004",53.8745709653744,5.52569020885136,"21/09/1801","00: 8:00",8,231,1801-09-21 00:08:00 +"1489173","Atlantis","1004",53.8770671329903,5.53651248163078,"21/09/1801"," 2:02:00",7,79,1801-09-21 02:02:00 +"1489174","Atlantis","1004",53.8674251821097,5.44649115431681,"21/09/1801"," 3:58:00",4,59,1801-09-21 03:58:00 +"1489175","Atlantis","1004",53.8890498611685,5.55427154102828,"21/09/1801"," 5:52:00",4,269,1801-09-21 05:52:00 +"1489176","Atlantis","1004",53.9126509298701,5.36414184421208,"21/09/1801"," 7:48:00",4,300,1801-09-21 07:48:00 +"1493756","Atlantis","1004",53.9220546782175,5.19734405987803,"21/09/1801"," 9:44:00",3,256,1801-09-21 09:44:00 +"1493757","Atlantis","1004",53.9030618058606,5.21180047482625,"21/09/1801","11:38:00",4,97,1801-09-21 11:38:00 +"1493758","Atlantis","1004",53.8746700852816,5.48578213242814,"21/09/1801","13:34:00",6,281,1801-09-21 13:34:00 +"1493759","Atlantis","1004",53.8945356517788,5.47507692701649,"21/09/1801","17:24:00",3,104,1801-09-21 17:24:00 +"1493760","Atlantis","1004",53.8677338226954,5.54203094118833,"21/09/1801","19:20:00",9,258,1801-09-21 19:20:00 +"1493761","Atlantis","1004",53.7728471342791,5.04146906879358,"21/09/1801","21:14:00",9,244,1801-09-21 21:14:00 +"1493762","Atlantis","1004",53.6680449370253,4.7111601828821,"21/09/1801","23:10:00",12,168,1801-09-21 23:10:00 +"1493763","Atlantis","1004",53.3509487155816,4.87574424170423,"22/09/1801"," 1:04:00",9,269,1801-09-22 01:04:00 +"1493764","Atlantis","1004",53.1749855099181,4.66625632005092,"22/09/1801"," 3:00:00",12,169,1801-09-22 03:00:00 +"1493765","Atlantis","1004",52.9559768810533,4.77402996963076,"22/09/1801"," 4:56:00",0,160,1801-09-22 04:56:00 +"1493766","Atlantis","1004",52.9582675949638,4.78059387337603,"22/09/1801"," 6:50:00",5,334,1801-09-22 06:50:00 +"1493767","Atlantis","1004",53.0405508777346,4.8471540538827,"22/09/1801"," 8:46:00",5,284,1801-09-22 08:46:00 +"1493768","Atlantis","1004",53.0363904897068,4.85920494180359,"22/09/1801","10:40:00",0,15,1801-09-22 10:40:00 +"1493769","Atlantis","1004",53.0450095155849,4.86082073102891,"22/09/1801","12:36:00",0,15,1801-09-22 12:36:00 +"1493770","Atlantis","1004",53.038896302205,4.8459733800143,"22/09/1801","14:32:00",0,15,1801-09-22 14:32:00 +"1493771","Atlantis","1004",53.0413674629563,4.84377708098665,"22/09/1801","16:26:00",0,15,1801-09-22 16:26:00 +"1493819","Atlantis","1004",53.0424839479635,4.85790369552746,"26/09/1801","20:16:00",0,269,1801-09-26 20:16:00 +"1493820","Atlantis","1004",53.0458899974492,4.84896117741428,"26/09/1801","22:12:00",0,269,1801-09-26 22:12:00 +"1493821","Atlantis","1004",53.0386887827418,4.85392880304903,"27/09/1801","00: 6:00",0,269,1801-09-27 00:06:00 +"1493822","Atlantis","1004",53.0459254124733,4.86072373101208,"27/09/1801"," 2:02:00",0,269,1801-09-27 02:02:00 +"1493823","Atlantis","1004",53.0456877512224,4.84478079823498,"27/09/1801"," 3:58:00",0,269,1801-09-27 03:58:00 +"1493824","Atlantis","1004",53.0435075528123,4.85599908633809,"27/09/1801"," 5:52:00",0,269,1801-09-27 05:52:00 +"1493825","Atlantis","1004",53.0393261683029,4.85116349788103,"27/09/1801"," 7:48:00",0,269,1801-09-27 07:48:00 +"1493826","Atlantis","1004",53.1771236844067,4.74853564695827,"27/09/1801"," 9:42:00",12,13,1801-09-27 09:42:00 +"1493827","Atlantis","1004",53.516384563677,5.00236942001246,"27/09/1801","11:38:00",13,39,1801-09-27 11:38:00 +"1493828","Atlantis","1004",53.8479856063551,5.10022544643842,"27/09/1801","13:34:00",1,235,1801-09-27 13:34:00 +"1493829","Atlantis","1004",53.885575038251,5.42934975891002,"27/09/1801","15:28:00",3,94,1801-09-27 15:28:00 +"1498390","Atlantis","1004",53.8732316767797,5.40910662597418,"27/09/1801","17:24:00",5,282,1801-09-27 17:24:00 +"1498391","Atlantis","1004",53.9012167348964,5.7450635034591,"27/09/1801","19:18:00",7,90,1801-09-27 19:18:00 +"1498392","Atlantis","1004",53.9826825512056,5.55440173270274,"27/09/1801","21:14:00",7,342,1801-09-27 21:14:00 +"1498393","Atlantis","1004",54.1746558995764,5.51592316549458,"27/09/1801","23:10:00",7,358,1801-09-27 23:10:00 +"1498394","Atlantis","1004",54.3429850017154,5.5216509736022,"28/09/1801"," 1:04:00",8,270,1801-09-28 01:04:00 +"1498395","Atlantis","1004",54.5438841840178,5.43429128994607,"28/09/1801"," 3:00:00",7,331,1801-09-28 03:00:00 +"1498396","Atlantis","1004",54.6265176144666,5.17080919660069,"28/09/1801"," 4:56:00",5,253,1801-09-28 04:56:00 +"1498397","Atlantis","1004",54.6130036303354,5.14441204735357,"28/09/1801"," 6:50:00",7,92,1801-09-28 06:50:00 +"1498398","Atlantis","1004",54.6442671393706,5.07349199787155,"28/09/1801"," 8:46:00",7,96,1801-09-28 08:46:00 +"1498399","Atlantis","1004",54.6556689588758,5.08711787709314,"28/09/1801","10:40:00",7,269,1801-09-28 10:40:00 +"1498400","Atlantis","1004",54.6749579561036,5.23317885142658,"28/09/1801","12:36:00",7,60,1801-09-28 12:36:00 +"1498401","Atlantis","1004",54.6885401250087,5.37116418926045,"28/09/1801","14:32:00",7,63,1801-09-28 14:32:00 +"1498402","Atlantis","1004",54.6870078116669,5.27801883398648,"28/09/1801","16:26:00",7,100,1801-09-28 16:26:00 +"1498403","Atlantis","1004",54.6775877113487,5.15577065772656,"28/09/1801","18:22:00",5,80,1801-09-28 18:22:00 +"1498404","Atlantis","1004",54.722408869857,5.09585706325993,"28/09/1801","20:16:00",8,263,1801-09-28 20:16:00 +"1498405","Atlantis","1004",54.554771568595,5.15765452293772,"28/09/1801","22:12:00",7,179,1801-09-28 22:12:00 +"1498406","Atlantis","1004",54.5472746419385,4.9549135602545,"29/09/1801"," 2:02:00",7,260,1801-09-29 02:02:00 +"1498407","Atlantis","1004",54.4777533122655,4.833164861314,"29/09/1801"," 3:58:00",7,224,1801-09-29 03:58:00 +"1498408","Atlantis","1004",54.5357829391104,4.99646874918137,"29/09/1801"," 5:52:00",7,75,1801-09-29 05:52:00 +"1498409","Atlantis","1004",54.6298046897952,5.30347497604322,"29/09/1801"," 7:48:00",8,54,1801-09-29 07:48:00 +"1498410","Atlantis","1004",54.7311634111167,5.55805744402949,"29/09/1801"," 9:44:00",7,79,1801-09-29 09:44:00 +"1498411","Atlantis","1004",54.7383905305443,5.57868081925437,"29/09/1801"," 9:46:00",7,79,1801-09-29 09:46:00 +"1498412","Atlantis","1004",54.860246430621,5.72348624049779,"29/09/1801","11:38:00",5,154,1801-09-29 11:38:00 +"1498413","Atlantis","1004",54.7078279539039,5.95806460902467,"29/09/1801","13:34:00",7,94,1801-09-29 13:34:00 +"1498414","Atlantis","1004",54.7347290559299,5.91020815948211,"29/09/1801","15:28:00",7,85,1801-09-29 15:28:00 +"1498415","Atlantis","1004",54.7171370475665,5.91121322342381,"29/09/1801","17:24:00",8,265,1801-09-29 17:24:00 +"1498416","Atlantis","1004",54.7189302717722,5.90433417737949,"29/09/1801","19:18:00",7,5,1801-09-29 19:18:00 +"1498417","Atlantis","1004",54.8243315741261,5.76416948076803,"29/09/1801","21:14:00",4,156,1801-09-29 21:14:00 +"1498418","Atlantis","1004",54.7710343252285,6.09365564829856,"29/09/1801","23:10:00",7,113,1801-09-29 23:10:00 +"1498419","Atlantis","1004",54.8287549877102,5.99204119027685,"30/09/1801"," 1:04:00",7,25,1801-09-30 01:04:00 +"1498420","Atlantis","1004",54.8456141366963,5.99877673429437,"30/09/1801"," 3:00:00",7,11,1801-09-30 03:00:00 +"1498421","Atlantis","1004",54.7943309591771,5.88489189432747,"30/09/1801"," 4:54:00",7,335,1801-09-30 04:54:00 +"1498422","Atlantis","1004",54.8511111874399,5.65373958104662,"30/09/1801"," 6:50:00",7,207,1801-09-30 06:50:00 +"1498423","Atlantis","1004",54.6627870871769,5.64673999824654,"30/09/1801"," 8:46:00",7,203,1801-09-30 08:46:00 +"1498424","Atlantis","1004",54.4420913820355,5.59335672736168,"30/09/1801","10:40:00",7,174,1801-09-30 10:40:00 +"1498425","Atlantis","1004",54.3057493562344,5.12166182311997,"30/09/1801","14:32:00",7,306,1801-09-30 14:32:00 +"1498426","Atlantis","1004",54.3742666170541,5.04340487123001,"30/09/1801","16:26:00",7,183,1801-09-30 16:26:00 +"1498427","Atlantis","1004",54.3740860519977,4.98883475052286,"30/09/1801","18:22:00",8,196,1801-09-30 18:22:00 +"1498428","Atlantis","1004",54.180676986238,4.9536893641809,"30/09/1801","20:16:00",7,188,1801-09-30 20:16:00 +"1498429","Atlantis","1004",53.9533367969156,4.83463261436485,"30/09/1801","22:12:00",7,199,1801-09-30 22:12:00 +"1498430","Atlantis","1004",53.7830785151166,4.69759559225664,"01/10/1801","00: 8:00",7,211,1801-10-01 00:08:00 +"1498431","Atlantis","1004",53.5950123654883,4.60024732373096,"01/10/1801"," 2:02:00",7,204,1801-10-01 02:02:00 +"1498432","Atlantis","1004",53.3870545951696,4.49037663356774,"01/10/1801"," 3:58:00",2,275,1801-10-01 03:58:00 +"1498433","Atlantis","1004",53.1399426974538,4.65807567935437,"01/10/1801"," 5:52:00",13,172,1801-10-01 05:52:00 +"1498434","Atlantis","1004",52.956249852933,4.78435198619682,"01/10/1801"," 7:48:00",0,174,1801-10-01 07:48:00 +"1498435","Atlantis","1004",52.9526760224863,4.76897201792058,"01/10/1801"," 9:42:00",0,174,1801-10-01 09:42:00 +"1498436","Atlantis","1004",53.0423439018903,4.84582812025957,"01/10/1801","13:34:00",0,286,1801-10-01 13:34:00 +"1498437","Atlantis","1004",53.0374628454484,4.85809543730877,"01/10/1801","15:28:00",0,14,1801-10-01 15:28:00 +"1498438","Atlantis","1004",53.0369052168787,4.86236234583612,"01/10/1801","17:24:00",0,14,1801-10-01 17:24:00 +"1498439","Atlantis","1004",53.0389626972415,4.86214114801679,"01/10/1801","19:20:00",0,14,1801-10-01 19:20:00 +"764928","Atlantis","1013",53.0445871164692,4.86463103540148,"01/06/1800","20:54:00",0,7,1800-06-01 20:54:00 +"764929","Atlantis","1013",53.0460339151365,4.86373789428174,"01/06/1800","21:32:00",0,7,1800-06-01 21:32:00 +"764930","Atlantis","1013",53.045582758402,4.85218078402244,"01/06/1800","21:52:00",0,7,1800-06-01 21:52:00 +"764931","Atlantis","1013",53.0386538732117,4.85375051884819,"01/06/1800","22:54:00",0,7,1800-06-01 22:54:00 +"764932","Atlantis","1013",53.0454681639113,4.85509372117836,"01/06/1800","23:28:00",0,7,1800-06-01 23:28:00 +"764933","Atlantis","1013",53.0464968574252,4.85186548210587,"01/06/1800","23:52:00",0,7,1800-06-01 23:52:00 +"764934","Atlantis","1013",53.0436705400185,4.84986880359613,"02/06/1800","00:54:00",0,7,1800-06-02 00:54:00 +"713052","Atlantis","1013",53.0469179774001,4.86592301778868,"02/06/1800"," 1:22:00",0,7,1800-06-02 01:22:00 +"713053","Atlantis","1013",53.0474315711665,4.84872884988226,"02/06/1800"," 1:54:00",0,7,1800-06-02 01:54:00 +"713054","Atlantis","1013",53.0385141079077,4.85337242881674,"02/06/1800"," 2:54:00",0,7,1800-06-02 02:54:00 +"713055","Atlantis","1013",53.0457420870517,4.85971545580216,"02/06/1800"," 3:18:00",0,7,1800-06-02 03:18:00 +"713056","Atlantis","1013",53.0467639279803,4.85813313893415,"02/06/1800"," 3:54:00",0,7,1800-06-02 03:54:00 +"713057","Atlantis","1013",53.0469054077631,4.85113428926002,"02/06/1800"," 4:54:00",0,7,1800-06-02 04:54:00 +"713058","Atlantis","1013",53.0453492441652,4.86283408027515,"02/06/1800"," 5:14:00",0,7,1800-06-02 05:14:00 +"713059","Atlantis","1013",53.047797080548,4.84913006685581,"02/06/1800"," 5:54:00",0,7,1800-06-02 05:54:00 +"713060","Atlantis","1013",53.0466775328568,4.84897106715851,"02/06/1800"," 6:54:00",0,7,1800-06-02 06:54:00 +"713061","Atlantis","1013",53.0401929945913,4.8545125005804,"02/06/1800"," 7:08:00",0,7,1800-06-02 07:08:00 +"713062","Atlantis","1013",53.0403545004032,4.86420177006069,"02/06/1800"," 7:54:00",0,7,1800-06-02 07:54:00 +"713063","Atlantis","1013",53.0034476027573,4.80379352284968,"02/06/1800"," 8:54:00",10,222,1800-06-02 08:54:00 +"713064","Atlantis","1013",52.9855472481167,4.77436282677576,"02/06/1800"," 9:04:00",9,233,1800-06-02 09:04:00 +"713065","Atlantis","1013",53.0727167975032,4.66835210822243,"02/06/1800"," 9:54:00",11,327,1800-06-02 09:54:00 +"713066","Atlantis","1013",53.2178393587619,4.48628928215336,"02/06/1800","10:54:00",11,324,1800-06-02 10:54:00 +"713067","Atlantis","1013",53.3641471679481,4.31758947358839,"02/06/1800","11:54:00",11,330,1800-06-02 11:54:00 +"713068","Atlantis","1013",53.500557712812,4.16456363636069,"02/06/1800","12:54:00",10,335,1800-06-02 12:54:00 +"713069","Atlantis","1013",53.5113623653916,4.15619658119418,"02/06/1800","12:56:00",10,331,1800-06-02 12:56:00 +"713070","Atlantis","1013",53.6418743312932,4.00462705611996,"02/06/1800","13:54:00",7,31,1800-06-02 13:54:00 +"713071","Atlantis","1013",53.6839725131998,4.08100425949414,"02/06/1800","14:50:00",4,53,1800-06-02 14:50:00 +"713072","Atlantis","1013",53.6884169258093,4.08944364615716,"02/06/1800","14:54:00",4,59,1800-06-02 14:54:00 +"713073","Atlantis","1013",53.7246389868674,4.18202278881054,"02/06/1800","15:54:00",4,57,1800-06-02 15:54:00 +"713074","Atlantis","1013",53.7574875975703,4.25733833231777,"02/06/1800","16:44:00",4,56,1800-06-02 16:44:00 +"713075","Atlantis","1013",53.7668795018671,4.27778346516564,"02/06/1800","16:54:00",4,55,1800-06-02 16:54:00 +"713076","Atlantis","1013",53.8127548598545,4.33073110030033,"02/06/1800","17:54:00",2,26,1800-06-02 17:54:00 +"713077","Atlantis","1013",53.8369536469337,4.39240467538312,"02/06/1800","18:40:00",4,48,1800-06-02 18:40:00 +"713078","Atlantis","1013",53.8510140035497,4.41071273786668,"02/06/1800","18:54:00",4,50,1800-06-02 18:54:00 +"713079","Atlantis","1013",53.8924040693426,4.47646900190599,"02/06/1800","19:54:00",4,11,1800-06-02 19:54:00 +"713080","Atlantis","1013",53.9384862843608,4.49087864725944,"02/06/1800","20:36:00",4,345,1800-06-02 20:36:00 +"713081","Atlantis","1013",54.0248097582464,4.47662326706853,"02/06/1800","21:54:00",4,359,1800-06-02 21:54:00 +"713082","Atlantis","1013",54.0539242369477,4.47393683302123,"02/06/1800","22:30:00",2,48,1800-06-02 22:30:00 +"713083","Atlantis","1013",54.0593657210646,4.46216298739892,"02/06/1800","22:54:00",4,271,1800-06-02 22:54:00 +"713084","Atlantis","1013",54.0724557867432,4.35473273146153,"02/06/1800","23:54:00",4,295,1800-06-02 23:54:00 +"713085","Atlantis","1013",54.0937221106337,4.31170431319065,"03/06/1800","00:26:00",4,298,1800-06-03 00:26:00 +"713086","Atlantis","1013",54.1054944752567,4.25656835188344,"03/06/1800","00:54:00",4,325,1800-06-03 00:54:00 +"713087","Atlantis","1013",54.1705195882171,4.24435283387452,"03/06/1800"," 1:54:00",4,0,1800-06-03 01:54:00 +"713088","Atlantis","1013",54.1975776973614,4.22664754016046,"03/06/1800"," 2:20:00",4,357,1800-06-03 02:20:00 +"713089","Atlantis","1013",54.2366604352007,4.23313521215413,"03/06/1800"," 2:54:00",5,65,1800-06-03 02:54:00 +"713090","Atlantis","1013",54.2204566313564,4.29476330519002,"03/06/1800"," 3:54:00",5,126,1800-06-03 03:54:00 +"713091","Atlantis","1013",54.1669422668009,4.36375270065013,"03/06/1800"," 4:54:00",4,174,1800-06-03 04:54:00 +"713092","Atlantis","1013",54.0995090839206,4.36887693024613,"03/06/1800"," 5:54:00",4,170,1800-06-03 05:54:00 +"713093","Atlantis","1013",54.0812382086078,4.38327260411251,"03/06/1800"," 6:10:00",4,175,1800-06-03 06:10:00 +"713094","Atlantis","1013",54.0545888009551,4.42777723629028,"03/06/1800"," 6:54:00",4,97,1800-06-03 06:54:00 +"713095","Atlantis","1013",54.1262567106523,4.46087200947013,"03/06/1800"," 8:06:00",3,1,1800-06-03 08:06:00 +"713096","Atlantis","1013",54.0963784918478,4.46367096793186,"03/06/1800"," 8:54:00",4,181,1800-06-03 08:54:00 +"713097","Atlantis","1013",54.0613508963357,4.42303086384758,"03/06/1800"," 9:54:00",4,278,1800-06-03 09:54:00 +"713098","Atlantis","1013",54.0543557870337,4.4176028245287,"03/06/1800","10:02:00",0,0,1800-06-03 10:02:00 +"713099","Atlantis","1013",54.1073659311994,4.37388325918093,"03/06/1800","10:54:00",4,346,1800-06-03 10:54:00 +"713100","Atlantis","1013",54.1567246059137,4.42744875764754,"03/06/1800","11:54:00",4,74,1800-06-03 11:54:00 +"713101","Atlantis","1013",54.1540674390523,4.44945978038013,"03/06/1800","11:56:00",4,72,1800-06-03 11:56:00 +"713102","Atlantis","1013",54.1309240520871,4.48307388088107,"03/06/1800","12:54:00",5,203,1800-06-03 12:54:00 +"713103","Atlantis","1013",54.0610545323333,4.4713629917521,"03/06/1800","13:52:00",4,180,1800-06-03 13:52:00 +"713104","Atlantis","1013",54.0594972396912,4.4586311679706,"03/06/1800","13:54:00",4,181,1800-06-03 13:54:00 +"713105","Atlantis","1013",53.9904294324801,4.46561224362068,"03/06/1800","14:54:00",4,175,1800-06-03 14:54:00 +"713106","Atlantis","1013",53.9322131360965,4.49262967333104,"03/06/1800","15:46:00",4,161,1800-06-03 15:46:00 +"713107","Atlantis","1013",53.9318582300474,4.48370686313696,"03/06/1800","15:56:00",4,175,1800-06-03 15:56:00 +"713108","Atlantis","1013",53.8810512356898,4.46390603309311,"03/06/1800","16:56:00",0,291,1800-06-03 16:56:00 +"713109","Atlantis","1013",53.9331506388229,4.48167346574925,"03/06/1800","17:42:00",4,351,1800-06-03 17:42:00 +"713110","Atlantis","1013",53.9497527976078,4.488143903994,"03/06/1800","17:54:00",4,341,1800-06-03 17:54:00 +"713111","Atlantis","1013",54.0151343890261,4.46992781254835,"03/06/1800","18:56:00",4,355,1800-06-03 18:56:00 +"713112","Atlantis","1013",54.0610182500887,4.45706815000065,"03/06/1800","19:38:00",4,4,1800-06-03 19:38:00 +"713113","Atlantis","1013",54.0798506334983,4.46390224244911,"03/06/1800","19:54:00",4,7,1800-06-03 19:54:00 +"713114","Atlantis","1013",54.1449411457027,4.46169580347464,"03/06/1800","20:56:00",4,339,1800-06-03 20:56:00 +"713115","Atlantis","1013",54.1783779362664,4.47027558408677,"03/06/1800","21:32:00",4,4,1800-06-03 21:32:00 +"713116","Atlantis","1013",54.2072464842112,4.47279666218813,"03/06/1800","21:56:00",4,359,1800-06-03 21:56:00 +"713117","Atlantis","1013",54.2684494530074,4.45861996958871,"03/06/1800","22:56:00",4,5,1800-06-03 22:56:00 +"713118","Atlantis","1013",54.3044780273423,4.4714724832084,"03/06/1800","23:28:00",4,5,1800-06-03 23:28:00 +"713119","Atlantis","1013",54.3290568605065,4.49945971040707,"03/06/1800","23:56:00",4,73,1800-06-03 23:56:00 +"713120","Atlantis","1013",54.3092585720257,4.61139908613078,"04/06/1800","00:56:00",4,127,1800-06-04 00:56:00 +"713121","Atlantis","1013",54.2854070612853,4.6452328699939,"04/06/1800"," 1:26:00",4,124,1800-06-04 01:26:00 +"713122","Atlantis","1013",54.3234183424842,4.57637164594792,"04/06/1800"," 2:56:00",4,306,1800-06-04 02:56:00 +"713123","Atlantis","1013",54.3343880567062,4.5462838120209,"04/06/1800"," 3:18:00",4,265,1800-06-04 03:18:00 +"713124","Atlantis","1013",54.3190655451552,4.48215522045642,"04/06/1800"," 3:56:00",4,198,1800-06-04 03:56:00 +"713125","Atlantis","1013",54.2534568882696,4.4595540603064,"04/06/1800"," 4:56:00",4,178,1800-06-04 04:56:00 +"764935","Atlantis","1013",54.2255252670785,4.45997055232898,"04/06/1800"," 5:14:00",4,181,1800-06-04 05:14:00 +"764936","Atlantis","1013",54.1827902029455,4.4608511306718,"04/06/1800"," 5:56:00",4,178,1800-06-04 05:56:00 +"764937","Atlantis","1013",54.1706070162682,4.46398532980122,"04/06/1800"," 6:56:00",4,10,1800-06-04 06:56:00 +"764938","Atlantis","1013",54.1866767612239,4.45307082518656,"04/06/1800"," 7:08:00",4,9,1800-06-04 07:08:00 +"764939","Atlantis","1013",54.2280650737668,4.49196241352055,"04/06/1800"," 7:56:00",4,190,1800-06-04 07:56:00 +"764940","Atlantis","1013",54.168189148712,4.45871613687836,"04/06/1800"," 8:56:00",4,181,1800-06-04 08:56:00 +"764941","Atlantis","1013",54.1581233380227,4.46346532505937,"04/06/1800"," 9:04:00",4,183,1800-06-04 09:04:00 +"764942","Atlantis","1013",54.0991315294104,4.46677847264335,"04/06/1800"," 9:56:00",4,184,1800-06-04 09:56:00 +"764943","Atlantis","1013",54.059838255534,4.46675562749244,"04/06/1800","10:56:00",0,184,1800-06-04 10:56:00 +"764944","Atlantis","1013",54.0678556052963,4.45623684887309,"04/06/1800","10:58:00",5,180,1800-06-04 10:58:00 +"764945","Atlantis","1013",54.0021821238678,4.46337098925188,"04/06/1800","11:56:00",4,179,1800-06-04 11:56:00 +"764946","Atlantis","1013",54.0226125252964,4.47464818236604,"04/06/1800","12:54:00",4,346,1800-06-04 12:54:00 +"764947","Atlantis","1013",54.0220069744927,4.46630089172907,"04/06/1800","12:56:00",4,346,1800-06-04 12:56:00 +"764948","Atlantis","1013",54.0765229104748,4.49666681366227,"04/06/1800","13:56:00",4,52,1800-06-04 13:56:00 +"764949","Atlantis","1013",54.1362038454637,4.50310239016358,"04/06/1800","14:56:00",2,357,1800-06-04 14:56:00 +"764950","Atlantis","1013",54.1131505484656,4.46757924990356,"04/06/1800","15:56:00",4,194,1800-06-04 15:56:00 +"764951","Atlantis","1013",54.0516472674878,4.46887763362378,"04/06/1800","16:44:00",4,179,1800-06-04 16:44:00 +"764952","Atlantis","1013",54.0385747598121,4.4678044912871,"04/06/1800","16:56:00",4,181,1800-06-04 16:56:00 +"764953","Atlantis","1013",53.9971018870021,4.48401023243088,"04/06/1800","17:56:00",0,182,1800-06-04 17:56:00 +"764954","Atlantis","1013",54.0431060008463,4.46214861033019,"04/06/1800","18:40:00",4,360,1800-06-04 18:40:00 +"764955","Atlantis","1013",54.0601452184292,4.45549645537045,"04/06/1800","18:56:00",4,359,1800-06-04 18:56:00 +"764956","Atlantis","1013",54.0924504707051,4.47381234739814,"04/06/1800","19:56:00",4,6,1800-06-04 19:56:00 +"764957","Atlantis","1013",54.1416818769146,4.46091814263724,"04/06/1800","20:34:00",4,2,1800-06-04 20:34:00 +"764958","Atlantis","1013",54.1626328017148,4.45924129400402,"04/06/1800","20:56:00",4,356,1800-06-04 20:56:00 +"764959","Atlantis","1013",54.2267736967779,4.45630392862018,"04/06/1800","21:56:00",4,1,1800-06-04 21:56:00 +"764960","Atlantis","1013",54.2712123377393,4.4624219927555,"04/06/1800","22:30:00",4,9,1800-06-04 22:30:00 +"764961","Atlantis","1013",54.3002301167794,4.48569389009755,"04/06/1800","22:56:00",4,8,1800-06-04 22:56:00 +"764962","Atlantis","1013",54.3280116153839,4.55928898821492,"04/06/1800","23:56:00",4,121,1800-06-04 23:56:00 +"764963","Atlantis","1013",54.3144435860305,4.60334784745145,"05/06/1800","00:26:00",3,124,1800-06-05 00:26:00 +"764964","Atlantis","1013",54.302965775717,4.62922692467924,"05/06/1800","00:56:00",4,308,1800-06-05 00:56:00 +"764965","Atlantis","1013",54.3263973832401,4.53139137225505,"05/06/1800"," 1:56:00",4,261,1800-06-05 01:56:00 +"764966","Atlantis","1013",54.3257364773415,4.48515264021419,"05/06/1800"," 2:20:00",4,209,1800-06-05 02:20:00 +"764967","Atlantis","1013",54.2888540545716,4.46404573750775,"05/06/1800"," 2:56:00",4,194,1800-06-05 02:56:00 diff --git a/tacsat_interpolated.csv b/tacsat_interpolated.csv new file mode 100644 index 0000000..535cb8e --- /dev/null +++ b/tacsat_interpolated.csv @@ -0,0 +1,813 @@ +"","VE_COU","VE_REF","SI_LATI","SI_LONG","SI_DATE","SI_TIME","SI_SP","SI_HE","SI_DATIM","HL_ID" +"658237","Atlantis","10",51.4477244459512,3.58373072574753,"04/05/1800","13:32:00",0,25,"1800-05-04 13:32:00",1 +"658238","Atlantis","10",51.4406743343966,3.58384667526838,"04/05/1800","15:28:00",0,25,"1800-05-04 15:28:00",2 +"662888","Atlantis","10",51.4407431283058,3.59552922427561,"04/05/1800","17:22:00",0,25,"1800-05-04 17:22:00",3 +"662889","Atlantis","10",51.4431543027703,3.58686242580321,"04/05/1800","19:18:00",0,25,"1800-05-04 19:18:00",4 +"662890","Atlantis","10",51.4435144856712,3.58694753077719,"04/05/1800","21:12:00",0,25,"1800-05-04 21:12:00",5 +"662891","Atlantis","10",51.4481111547514,3.59395834521763,"04/05/1800","23:08:00",0,25,"1800-05-04 23:08:00",6 +"662892","Atlantis","10",51.4474296509796,3.59117479477078,"05/05/1800"," 1:04:00",0,111,"1800-05-05 01:04:00",7 +"662893","Atlantis","10",51.6214494813131,3.19755943488237,"05/05/1800"," 2:58:00",12,292,"1800-05-05 02:58:00",8 +"662894","Atlantis","10",51.6731934304331,3.17657612829283,"05/05/1800"," 4:54:00",2,43,"1800-05-05 04:54:00",9 +"662895","Atlantis","10",51.6943234914839,3.22908421075065,"05/05/1800"," 6:48:00",2,294,"1800-05-05 06:48:00",10 +"662896","Atlantis","10",51.6560659951191,3.16098873873707,"05/05/1800"," 8:44:00",6,144,"1800-05-05 08:44:00",11 +"662897","Atlantis","10",51.6070417380915,3.08350550274644,"05/05/1800","10:40:00",7,248,"1800-05-05 10:40:00",12 +"662898","Atlantis","10",51.576919331173,2.98779570994805,"05/05/1800","14:30:00",6,228,"1800-05-05 14:30:00",13 +"662899","Atlantis","10",51.5465515189292,2.87639516843669,"05/05/1800","16:24:00",7,50,"1800-05-05 16:24:00",14 +"662900","Atlantis","10",51.6061289456477,2.97909264942538,"05/05/1800","18:20:00",7,330,"1800-05-05 18:20:00",15 +"662901","Atlantis","10",51.6016501706596,3.0204385781968,"05/05/1800","20:16:00",6,58,"1800-05-05 20:16:00",16 +"662902","Atlantis","10",51.5731208998202,2.8929834927395,"05/05/1800","22:10:00",7,253,"1800-05-05 22:10:00",17 +"662903","Atlantis","10",51.5931214836561,2.99407952163834,"06/05/1800","00: 6:00",7,245,"1800-05-06 00:06:00",18 +"662904","Atlantis","10",51.5683078272836,2.93339168617967,"06/05/1800"," 2:00:00",7,330,"1800-05-06 02:00:00",19 +"662905","Atlantis","10",51.5473403272065,2.87607746142615,"06/05/1800"," 3:56:00",7,55,"1800-05-06 03:56:00",20 +"662906","Atlantis","10",51.5662122405847,2.93659450235404,"06/05/1800"," 5:52:00",7,83,"1800-05-06 05:52:00",21 +"662907","Atlantis","10",51.6299493680494,3.036660480747,"06/05/1800"," 7:46:00",7,59,"1800-05-06 07:46:00",22 +"662908","Atlantis","10",51.6359223276689,3.09941987676174,"06/05/1800"," 9:42:00",6,116,"1800-05-06 09:42:00",23 +"662909","Atlantis","10",51.5344633132401,2.87679365528095,"06/05/1800","11:36:00",7,228,"1800-05-06 11:36:00",24 +"662910","Atlantis","10",51.511504783609,2.31121867679339,"06/05/1800","13:32:00",14,271,"1800-05-06 13:32:00",25 +"662911","Atlantis","10",51.4578128865957,1.82871440000553,"06/05/1800","15:28:00",6,172,"1800-05-06 15:28:00",26 +"662912","Atlantis","10",51.5112028607992,1.8587873764392,"06/05/1800","17:22:00",7,25,"1800-05-06 17:22:00",27 +"662913","Atlantis","10",51.6795650937799,1.95230285054259,"06/05/1800","19:18:00",7,20,"1800-05-06 19:18:00",28 +"662914","Atlantis","10",51.695745846319,1.92481063032057,"06/05/1800","21:12:00",7,102,"1800-05-06 21:12:00",29 +"662915","Atlantis","10",51.5305358059094,1.86397848288994,"06/05/1800","23:08:00",7,204,"1800-05-06 23:08:00",30 +"662916","Atlantis","10",51.4609367726953,1.83341799937282,"07/05/1800"," 1:04:00",6,12,"1800-05-07 01:04:00",31 +"662917","Atlantis","10",51.4061924462626,1.80183325022925,"07/05/1800"," 2:58:00",6,191,"1800-05-07 02:58:00",32 +"662918","Atlantis","10",51.4650244918605,1.83690901399869,"07/05/1800"," 4:54:00",7,12,"1800-05-07 04:54:00",33 +"662919","Atlantis","10",51.5882227651426,2.02424114330858,"07/05/1800"," 6:48:00",13,123,"1800-05-07 06:48:00",34 +"662920","Atlantis","10",51.4148697609562,2.18524577266816,"07/05/1800"," 8:44:00",6,223,"1800-05-07 08:44:00",35 +"662921","Atlantis","10",51.4457611120637,2.22457806055807,"07/05/1800","12:34:00",7,218,"1800-05-07 12:34:00",36 +"662922","Atlantis","10",51.4589082118147,2.2554634172162,"07/05/1800","14:30:00",9,301,"1800-05-07 14:30:00",37 +"662923","Atlantis","10",51.4187489186181,2.19479238274507,"07/05/1800","16:24:00",6,215,"1800-05-07 16:24:00",38 +"662924","Atlantis","10",51.5047392145819,2.39375767114479,"07/05/1800","18:20:00",11,138,"1800-05-07 18:20:00",39 +"662925","Atlantis","10",51.4934193430715,2.42827651779447,"08/05/1800"," 2:00:00",7,229,"1800-05-08 02:00:00",40 +"662926","Atlantis","10",51.4012685797266,2.24624043547455,"08/05/1800"," 3:56:00",6,38,"1800-05-08 03:56:00",41 +"662927","Atlantis","10",51.478767084531,2.40182553862966,"08/05/1800"," 5:52:00",5,230,"1800-05-08 05:52:00",42 +"662928","Atlantis","10",51.4856400817088,2.40774057370703,"08/05/1800"," 7:46:00",7,43,"1800-05-08 07:46:00",43 +"662929","Atlantis","10",51.5109846706307,2.51050425128173,"08/05/1800"," 9:42:00",6,203,"1800-05-08 09:42:00",44 +"662930","Atlantis","10",51.3685016771392,2.30006208344549,"08/05/1800","11:36:00",6,191,"1800-05-08 11:36:00",45 +"662931","Atlantis","10",51.4594513731385,2.34565509427059,"08/05/1800","13:32:00",7,236,"1800-05-08 13:32:00",46 +"662932","Atlantis","10",51.4663128634985,2.3640575841507,"08/05/1800","17:22:00",7,56,"1800-05-08 17:22:00",47 +"662933","Atlantis","10",51.3029056911548,2.4005533228144,"08/05/1800","19:18:00",7,37,"1800-05-08 19:18:00",48 +"662934","Atlantis","10",51.3948131802687,2.64025816989597,"08/05/1800","21:12:00",7,74,"1800-05-08 21:12:00",49 +"662935","Atlantis","10",51.4710751335397,2.92117160728946,"08/05/1800","23:08:00",11,136,"1800-05-08 23:08:00",50 +"662936","Atlantis","10",51.4205238452363,3.47438484901283,"09/05/1800"," 1:04:00",11,82,"1800-05-09 01:04:00",51 +"662937","Atlantis","10",51.4485953284753,3.59852260938287,"09/05/1800"," 2:58:00",0,15,"1800-05-09 02:58:00",52 +"662938","Atlantis","10",51.4467330461042,3.58989608570375,"09/05/1800"," 4:54:00",0,15,"1800-05-09 04:54:00",53 +"662939","Atlantis","10",51.4472292818362,3.5884271329958,"09/05/1800"," 6:48:00",0,15,"1800-05-09 06:48:00",54 +"662940","Atlantis","10",51.4471316511449,3.58982776055485,"09/05/1800"," 8:44:00",0,15,"1800-05-09 08:44:00",55 +"662941","Atlantis","10",51.4484958911571,3.58411605070718,"09/05/1800","12:34:00",0,15,"1800-05-09 12:34:00",56 +"667604","Atlantis","10",51.4437598910858,3.583766422024,"11/05/1800","14:30:00",0,15,"1800-05-11 14:30:00",57 +"667605","Atlantis","10",51.4416034219647,3.5851950481059,"11/05/1800","16:24:00",0,15,"1800-05-11 16:24:00",58 +"667606","Atlantis","10",51.4472780093341,3.59583351112343,"11/05/1800","18:20:00",0,15,"1800-05-11 18:20:00",59 +"667607","Atlantis","10",51.4445924110152,3.59097028739192,"11/05/1800","20:16:00",0,15,"1800-05-11 20:16:00",60 +"667608","Atlantis","10",51.4429707641923,3.59499000045843,"11/05/1800","22:10:00",0,15,"1800-05-11 22:10:00",61 +"667609","Atlantis","10",51.4448838578933,3.59818694050796,"12/05/1800","00: 6:00",0,15,"1800-05-12 00:06:00",62 +"667610","Atlantis","10",51.4455564921806,3.55852451810613,"12/05/1800"," 2:00:00",14,311,"1800-05-12 02:00:00",63 +"667611","Atlantis","10",51.6548137522223,3.14251614461281,"12/05/1800"," 5:52:00",7,218,"1800-05-12 05:52:00",64 +"667612","Atlantis","10",51.5775022460828,2.90554459167179,"12/05/1800"," 9:42:00",7,60,"1800-05-12 09:42:00",65 +"667613","Atlantis","10",51.6453192323837,2.95586224767007,"12/05/1800","11:36:00",7,336,"1800-05-12 11:36:00",66 +"667614","Atlantis","10",51.911236426278,2.95896811183542,"12/05/1800","13:32:00",13,349,"1800-05-12 13:32:00",67 +"667615","Atlantis","10",51.9453403444826,2.94993140023574,"12/05/1800","13:42:00",13,342,"1800-05-12 13:42:00",68 +"667616","Atlantis","10",52.0593325713198,2.73492714592721,"12/05/1800","15:28:00",7,309,"1800-05-12 15:28:00",69 +"667617","Atlantis","10",52.0947284897235,2.43497160246968,"12/05/1800","17:22:00",6,262,"1800-05-12 17:22:00",70 +"667618","Atlantis","10",51.9565004344275,2.27307889834326,"12/05/1800","19:18:00",14,216,"1800-05-12 19:18:00",71 +"667619","Atlantis","10",51.6590415419177,1.9370064862268,"12/05/1800","21:12:00",6,196,"1800-05-12 21:12:00",72 +"667620","Atlantis","10",51.4842405747063,1.84669348298386,"12/05/1800","23:08:00",5,197,"1800-05-12 23:08:00",73 +"667621","Atlantis","10",51.5091452060263,1.85640532855037,"13/05/1800"," 1:04:00",7,26,"1800-05-13 01:04:00",74 +"667622","Atlantis","10",51.4929940155745,1.83951853084937,"13/05/1800"," 2:58:00",6,204,"1800-05-13 02:58:00",75 +"667623","Atlantis","10",51.4471113794856,1.81924429896846,"13/05/1800"," 4:54:00",6,190,"1800-05-13 04:54:00",76 +"667624","Atlantis","10",51.4249639321375,1.80902313761879,"13/05/1800"," 6:48:00",5,18,"1800-05-13 06:48:00",77 +"667625","Atlantis","10",51.4693282687338,1.84038470214791,"13/05/1800"," 8:44:00",5,10,"1800-05-13 08:44:00",78 +"667626","Atlantis","10",51.4418218042934,1.81663903429546,"13/05/1800","10:40:00",7,11,"1800-05-13 10:40:00",79 +"667627","Atlantis","10",51.4088726234972,2.16847054663394,"13/05/1800","12:34:00",8,50,"1800-05-13 12:34:00",80 +"667628","Atlantis","10",51.5362964495379,2.39018587551173,"13/05/1800","14:30:00",5,304,"1800-05-13 14:30:00",81 +"667629","Atlantis","10",51.5190567561476,2.35887411012128,"13/05/1800","16:24:00",1,341,"1800-05-13 16:24:00",82 +"667630","Atlantis","10",51.4108731082259,2.18432660974283,"13/05/1800","18:20:00",5,52,"1800-05-13 18:20:00",83 +"667631","Atlantis","10",51.2596449704305,2.06133533269446,"13/05/1800","20:16:00",7,218,"1800-05-13 20:16:00",84 +"667632","Atlantis","10",51.2466653574975,2.03709894624539,"13/05/1800","22:10:00",6,234,"1800-05-13 22:10:00",85 +"667633","Atlantis","10",51.3371365424073,2.19485791365616,"14/05/1800"," 2:00:00",7,100,"1800-05-14 02:00:00",86 +"667634","Atlantis","10",51.3545975816706,2.50843999481108,"14/05/1800"," 3:56:00",1,77,"1800-05-14 03:56:00",87 +"667635","Atlantis","10",51.4250464622211,2.76797837713175,"14/05/1800"," 5:52:00",6,69,"1800-05-14 05:52:00",88 +"667636","Atlantis","10",51.4208011545157,2.72131891132705,"14/05/1800"," 7:46:00",5,78,"1800-05-14 07:46:00",89 +"667637","Atlantis","10",51.4992059450257,2.95690686668456,"14/05/1800"," 9:42:00",8,47,"1800-05-14 09:42:00",90 +"667638","Atlantis","10",51.5439986988604,3.03306273283996,"14/05/1800","11:36:00",7,62,"1800-05-14 11:36:00",91 +"667639","Atlantis","10",51.5604905423173,3.07679762857966,"14/05/1800","13:32:00",6,262,"1800-05-14 13:32:00",92 +"667640","Atlantis","10",51.5730975413276,3.0872489113044,"14/05/1800","15:28:00",7,332,"1800-05-14 15:28:00",93 +"667641","Atlantis","10",51.5740948810196,3.08704559132829,"14/05/1800","17:22:00",6,330,"1800-05-14 17:22:00",94 +"667642","Atlantis","10",51.5730116140666,3.06881503879931,"14/05/1800","19:18:00",6,72,"1800-05-14 19:18:00",95 +"667643","Atlantis","10",51.5742488432918,3.07494611991662,"14/05/1800","21:12:00",5,54,"1800-05-14 21:12:00",96 +"667644","Atlantis","10",51.562859330554,3.00578095217701,"14/05/1800","23:08:00",7,242,"1800-05-14 23:08:00",97 +"667645","Atlantis","10",51.5095643621851,2.87637615688983,"15/05/1800"," 1:04:00",7,67,"1800-05-15 01:04:00",98 +"667646","Atlantis","10",51.566353250348,3.03543414315209,"15/05/1800"," 2:58:00",5,241,"1800-05-15 02:58:00",99 +"667647","Atlantis","10",51.5579811066706,3.01812346521858,"15/05/1800"," 4:54:00",7,73,"1800-05-15 04:54:00",100 +"22","Atlantis","10",51.4469073573065,3.58374416397983,"NA ","NA ",0,179.412723830397,NA,1 +"31","Atlantis","10",51.4461746259528,3.58375621483161,"NA ","NA ",0,179.412713942172,NA,1 +"39","Atlantis","10",51.4454057176007,3.58376886066769,"NA ","NA ",0,179.412703228689,NA,1 +"47","Atlantis","10",51.4445726366737,3.58378256191675,"NA ","NA ",0,179.412693628994,NA,1 +"54","Atlantis","10",51.4438261436741,3.58379483909916,"NA ","NA ",0,179.412682915796,NA,1 +"62","Atlantis","10",51.4429930627471,3.58380854034822,"NA ","NA ",0,179.412673028031,NA,1 +"70","Atlantis","10",51.442224154395,3.5838211861843,"NA ","NA ",0,179.412663605549,NA,1 +"79","Atlantis","10",51.4414914230413,3.58383323703608,"NA ","NA ",0,179.412653098354,NA,1 +"221","Atlantis","10",51.4406823074225,3.58520065074255,"NA ","NA ",0,89.4582628146559,NA,2 +"311","Atlantis","10",51.4406894573029,3.58641484003331,"NA ","NA ",0,89.4582392882277,NA,2 +"391","Atlantis","10",51.4406969601929,3.58768897724889,"NA ","NA ",0,89.4581976198418,NA,2 +"471","Atlantis","10",51.440705089269,3.58906945310115,"NA ","NA ",0,89.4582536271917,NA,2 +"541","Atlantis","10",51.4407123734334,3.59030644644284,"NA ","NA ",0,89.4581974372188,NA,2 +"621","Atlantis","10",51.4407205025095,3.5916869222951,"NA ","NA ",0,89.4582389209097,NA,2 +"701","Atlantis","10",51.4407280053995,3.59296105951069,"NA ","NA ",0,89.4582622721609,NA,2 +"791","Atlantis","10",51.4407351552799,3.59417524880144,"NA ","NA ",0,89.4582075291148,NA,2 +"222","Atlantis","10",51.4410225768308,3.59452476603932,"NA ","NA ",0,294.053192039537,NA,3 +"312","Atlantis","10",51.4412731747279,3.59362400937364,"NA ","NA ",0,294.053329330289,NA,3 +"392","Atlantis","10",51.4415361453447,3.59267877982941,"NA ","NA ",0,294.05348808694,NA,3 +"472","Atlantis","10",51.4418210633126,3.59165466206194,"NA ","NA ",0,294.053572594947,NA,3 +"542","Atlantis","10",51.4420763677634,3.59073698801688,"NA ","NA ",0,294.053740341198,NA,3 +"622","Atlantis","10",51.4423612857314,3.58971287024941,"NA ","NA ",0,294.053837416066,NA,3 +"702","Atlantis","10",51.4426242563482,3.58876764070518,"NA ","NA ",0,294.053939935855,NA,3 +"792","Atlantis","10",51.4428748542453,3.5878668840395,"NA ","NA ",0,294.054104239218,NA,3 +"223","Atlantis","10",51.4431960469833,3.5868722892369,"NA ","NA ",0,8.37787212418925,NA,4 +"313","Atlantis","10",51.4432334814696,3.58688113435691,"NA ","NA ",0,8.37786501948943,NA,4 +"393","Atlantis","10",51.4432727642013,3.58689041618526,"NA ","NA ",0,8.3778573135678,NA,4 +"473","Atlantis","10",51.4433153254435,3.58690047266878,"NA ","NA ",0,8.37785042757957,NA,4 +"543","Atlantis","10",51.443353462998,3.58690948391162,"NA ","NA ",0,8.37784271867145,NA,4 +"623","Atlantis","10",51.4433960242402,3.58691954039514,"NA ","NA ",0,8.3778356228857,NA,4 +"703","Atlantis","10",51.4434353069719,3.58692882222349,"NA ","NA ",0,8.37782885488718,NA,4 +"793","Atlantis","10",51.4434727414582,3.5869376673435,"NA ","NA ",0,8.3778213003406,NA,4 +"224","Atlantis","10",51.4440472270441,3.58776006499386,"NA ","NA ",0,43.5494641140601,NA,5 +"314","Atlantis","10",51.444524967507,3.58848871212453,"NA ","NA ",0,43.5491433367004,NA,5 +"394","Atlantis","10",51.4450262953541,3.58925333460415,"NA ","NA ",0,43.5487911693164,NA,5 +"474","Atlantis","10",51.4455694636869,3.59008177196261,"NA ","NA ",0,43.5485021953525,NA,5 +"544","Atlantis","10",51.4460561767358,3.59082410403221,"NA ","NA ",0,43.5481458796077,NA,5 +"624","Atlantis","10",51.4465993450685,3.59165254139067,"NA ","NA ",0,43.5478435987286,NA,5 +"704","Atlantis","10",51.4471006729156,3.59241716387029,"NA ","NA ",0,43.5475509284576,NA,5 +"794","Atlantis","10",51.4475784133785,3.59314581100096,"NA ","NA ",0,43.5472015387114,NA,5 +"225","Atlantis","10",51.4480321703284,3.5936357393348,"NA ","NA ",0,248.552832149663,NA,6 +"315","Atlantis","10",51.4479613403596,3.59334643970158,"NA ","NA ",0,248.552868730297,NA,6 +"395","Atlantis","10",51.447887013317,3.59304285653548,"NA ","NA ",0,248.552911702991,NA,6 +"475","Atlantis","10",51.4478064829893,3.59271393652213,"NA ","NA ",0,248.552930933158,NA,6 +"545","Atlantis","10",51.4477343227417,3.59241920346628,"NA ","NA ",0,248.552976907889,NA,6 +"625","Atlantis","10",51.447653792414,3.59209028345293,"NA ","NA ",0,248.553000064348,NA,6 +"705","Atlantis","10",51.4475794653714,3.59178670028682,"NA ","NA ",0,248.553025473961,NA,6 +"795","Atlantis","10",51.4475086354026,3.59149740065361,"NA ","NA ",0,248.553070487012,NA,6 +"44","Atlantis","10",51.4801729381231,3.58132400100967,"NA ","NA ",6,317.579820101462,NA,7 +"59","Atlantis","10",51.507043588043,3.54186552427335,"NA ","NA ",6,307.650388859212,NA,7 +"68","Atlantis","10",51.5270135459673,3.50024513842088,"NA ","NA ",6,302.477715386282,NA,7 +"76","Atlantis","10",51.5471907891991,3.44924331781578,"NA ","NA ",6,299.098366312378,NA,7 +"82","Atlantis","10",51.5638238481176,3.40113949379055,"NA ","NA ",6,296.863347682025,NA,7 +"87","Atlantis","10",51.5786678270778,3.3539519486754,"NA ","NA ",6,295.094060520936,NA,7 +"92","Atlantis","10",51.5944061968316,3.29981131046736,"NA ","NA ",6,293.679407380722,NA,7 +"96","Atlantis","10",51.6076412740127,3.25117907496127,"NA ","NA ",6,292.545898884969,NA,7 +"5","Atlantis","10",51.6344973628352,3.14544849597458,"NA ","NA ",7,291.930135253937,NA,8 +"9","Atlantis","10",51.6455458983961,3.10119072029649,"NA ","NA ",7,291.907498513654,NA,8 +"14","Atlantis","10",51.6567612754601,3.05620160046936,"NA ","NA ",7,292.007110865511,NA,8 +"21","Atlantis","10",51.6681151650798,3.01087316183413,"NA ","NA ",7,297.825821209489,NA,8 +"45","Atlantis","10",51.6784377903751,2.97932010622738,"NA ","NA ",7,102.246164199678,NA,8 +"61","Atlantis","10",51.671834355703,3.02830139870105,"NA ","NA ",7,99.7317450294168,NA,8 +"73","Atlantis","10",51.6662618553922,3.08057850689918,"NA ","NA ",7,93.7121985543716,NA,8 +"84","Atlantis","10",51.6643349972901,3.12821903104384,"NA ","NA ",7,73.5256970506064,NA,8 +"7","Atlantis","10",51.6792633050023,3.18703813659989,"NA ","NA ",2,55.6825565550446,NA,9 +"141","Atlantis","10",51.6844818024859,3.19937052431885,"NA ","NA ",2,65.1606368359359,NA,9 +"211","Atlantis","10",51.6879649460299,3.21151030715707,"NA ","NA ",2,75.3860212180585,NA,9 +"30","Atlantis","10",51.6903512637361,3.2262795383551,"NA ","NA ",2,86.3329227519476,NA,9 +"40","Atlantis","10",51.6909254133314,3.24075332708715,"NA ","NA ",2,97.7765848380197,NA,9 +"53","Atlantis","10",51.6897255875378,3.25491549707973,"NA ","NA ",2,157.843442463725,NA,9 +"83","Atlantis","10",51.6880124665357,3.25604075301636,"NA ","NA ",2,288.48702378797,NA,9 +"93","Atlantis","10",51.6906997678377,3.24307156711292,"NA ","NA ",2,292.687336334909,NA,9 +"12","Atlantis","10",51.7026753800999,3.20186085596824,"NA ","NA ",4,298.489933356938,NA,10 +"23","Atlantis","10",51.712425434355,3.17285631303373,"NA ","NA ",4,298.017653892753,NA,10 +"34","Atlantis","10",51.7216241488096,3.1449400759293,"NA ","NA ",4,293.632873965216,NA,10 +"476","Atlantis","10",51.7291187985766,3.11727499019433,"NA ","NA ",4,240.050076551497,NA,10 +"72","Atlantis","10",51.7218868286621,3.09701855808414,"NA ","NA ",4,156.665517408791,NA,10 +"831","Atlantis","10",51.7049455005677,3.10881102342792,"NA ","NA ",4,148.895719353132,NA,10 +"90","Atlantis","10",51.6884640914905,3.12485013976496,"NA ","NA ",4,145.991459388113,NA,10 +"961","Atlantis","10",51.6703512649036,3.14455405260719,"NA ","NA ",4,144.481408221284,NA,10 +"71","Atlantis","10",51.636118826171,3.18654591545532,"NA ","NA ",6.5,134.624075149241,NA,11 +"15","Atlantis","10",51.6176582604411,3.21666400502882,"NA ","NA ",6.5,122.074643518618,NA,11 +"26","Atlantis","10",51.60486160824,3.24952984600151,"NA ","NA ",6.5,65.8047920659282,NA,11 +"546","Atlantis","10",51.6125608795955,3.27713589525904,"NA ","NA ",6.5,298.849518620603,NA,11 +"721","Atlantis","10",51.6246059582413,3.24190021367582,"NA ","NA ",6.5,272.949354759086,NA,11 +"821","Atlantis","10",51.6259093572166,3.20092591489644,"NA ","NA ",6.5,261.852548082096,NA,11 +"89","Atlantis","10",51.6224630772922,3.16222441669604,"NA ","NA ",6.5,254.96376832094,NA,11 +"95","Atlantis","10",51.6157611623899,3.12208590279149,"NA ","NA ",6.5,250.016420755288,NA,11 +"8","Atlantis","10",51.5558173542626,2.94905627284677,"NA ","NA ",6.5,230.314639693738,NA,13 +"151","Atlantis","10",51.5367531236584,2.91213107529111,"NA ","NA ",6.5,231.851866750858,NA,13 +"226","Atlantis","10",51.5199358174867,2.87773220203195,"NA ","NA ",6.5,233.737310880219,NA,13 +"316","Atlantis","10",51.5019518064606,2.83836320821451,"NA ","NA ",6.5,236.867860403308,NA,13 +"43","Atlantis","10",51.4850572268304,2.79681437554208,"NA ","NA ",6.5,269.038445915141,NA,13 +"731","Atlantis","10",51.4847727667176,2.7698925723191,"NA ","NA ",6.5,43.6309077448594,NA,13 +"86","Atlantis","10",51.5063865005802,2.80300352994433,"NA ","NA ",6.5,47.8665688050339,NA,13 +"94","Atlantis","10",51.5270908895428,2.83980125394159,"NA ","NA ",6.5,49.4563110507621,NA,13 +"6","Atlantis","10",51.5617773741094,2.90873400675969,"NA ","NA ",7,60.9921094166979,NA,14 +"13","Atlantis","10",51.5756288875948,2.94894934242562,"NA ","NA ",7,74.0137973616934,NA,14 +"212","Atlantis","10",51.58252975176,2.98775028681977,"NA ","NA ",7,94.7732234847981,NA,14 +"32","Atlantis","10",51.5803933011356,3.02878564388985,"NA ","NA ",7,125.922166488295,NA,14 +"477","Atlantis","10",51.5653696599212,3.06213541102408,"NA ","NA ",7,187.546034670719,NA,14 +"74","Atlantis","10",51.5465477178351,3.0581260548296,"NA ","NA ",7,309.056611510952,NA,14 +"871","Atlantis","10",51.5619231249431,3.02763777496138,"NA ","NA ",7,323.354967465678,NA,14 +"951","Atlantis","10",51.5851077847022,2.99987704110266,"NA ","NA ",7,328.449447403332,NA,14 +"75","Atlantis","10",51.6303988845848,2.95393662582827,"NA ","NA ",6.5,319.094103764917,NA,15 +"142","Atlantis","10",51.6488526500634,2.92816474361454,"NA ","NA ",6.5,294.423423829791,NA,15 +"28","Atlantis","10",51.6598437815857,2.88912455288068,"NA ","NA ",6.5,215.773871956705,NA,15 +"46","Atlantis","10",51.6400178095173,2.86610880136074,"NA ","NA ",6.5,172.367013204798,NA,15 +"60","Atlantis","10",51.6136143517696,2.87180710568054,"NA ","NA ",6.5,143.174395572326,NA,15 +"732","Atlantis","10",51.5923165160734,2.89747390010666,"NA ","NA ",6.5,107.131510006385,NA,15 +"85","Atlantis","10",51.5841679679655,2.93997815873451,"NA ","NA ",6.5,77.8343992972692,NA,15 +"941","Atlantis","10",51.5901192745179,2.98447500907465,"NA ","NA ",6.5,62.6859234308714,NA,15 +"81","Atlantis","10",51.6170540755326,3.06134204160413,"NA ","NA ",6.5,60.8154332738033,NA,16 +"17","Atlantis","10",51.6311786490415,3.10210361653911,"NA ","NA ",6.5,66.3560652033527,NA,16 +"35","Atlantis","10",51.6427241588264,3.1446279349412,"NA ","NA ",6.5,232.701137570912,NA,16 +"63","Atlantis","10",51.6269957711017,3.11137751265542,"NA ","NA ",6.5,243.530243236908,NA,16 +"733","Atlantis","10",51.6146176466485,3.07136469631603,"NA ","NA ",6.5,246.58324358565,NA,16 +"822","Atlantis","10",51.6016352870869,3.02313998234658,"NA ","NA ",6.5,248.765930188259,NA,16 +"88","Atlantis","10",51.5923469211521,2.98468781950761,"NA ","NA ",6.5,250.482429930731,NA,16 +"942","Atlantis","10",51.5827776058793,2.94127820094434,"NA ","NA ",6.5,252.183140979438,NA,16 +"77","Atlantis","10",51.5656217393729,2.8520364287813,"NA ","NA ",7,273.220492575541,NA,17 +"281","Atlantis","10",51.5662639845284,2.83363302437281,"NA ","NA ",7,67.227342558316,NA,17 +"396","Atlantis","10",51.5773835947189,2.87628487710894,"NA ","NA ",7,68.9643770334318,NA,17 +"478","Atlantis","10",51.5874406054005,2.91840555130997,"NA ","NA ",7,69.8765209481903,NA,17 +"547","Atlantis","10",51.5963951494415,2.95777530472053,"NA ","NA ",7,70.8519395456699,NA,17 +"626","Atlantis","10",51.6055862025497,3.00043160607566,"NA ","NA ",7,72.9716853834235,NA,17 +"734","Atlantis","10",51.6140069116365,3.04475283010871,"NA ","NA ",7,219.033326753342,NA,17 +"943","Atlantis","10",51.6046473468196,3.03253591933,"NA ","NA ",7,244.257660531301,NA,17 +"78","Atlantis","10",51.5789479910796,2.95253474888816,"NA ","NA ",7,230.960931195187,NA,18 +"143","Atlantis","10",51.5621321456373,2.91918710379016,"NA ","NA ",7,213.619130171772,NA,18 +"231","Atlantis","10",51.5410183236549,2.8966183781533,"NA ","NA ",7,180.916772705995,NA,18 +"37","Atlantis","10",51.5122034515242,2.89587747831734,"NA ","NA ",7,140.882436692577,NA,18 +"52","Atlantis","10",51.4917289062084,2.92261442877787,"NA ","NA ",7,90.6225371805211,NA,18 +"722","Atlantis","10",51.4914462036429,2.96335793916715,"NA ","NA ",7,7.49828072794611,NA,18 +"861","Atlantis","10",51.5166123883782,2.96868094100854,"NA ","NA ",7,341.487233116472,NA,18 +"944","Atlantis","10",51.5425875616167,2.95469568992458,"NA ","NA ",7,332.759916739484,NA,18 +"710","Atlantis","10",51.5921992649867,2.90691389442711,"NA ","NA ",7,309.888366563184,NA,19 +"16","Atlantis","10",51.612306700815,2.86815934132051,"NA ","NA ",7,276.324476980657,NA,19 +"282","Atlantis","10",51.615539725598,2.8210446272726,"NA ","NA ",7,235.291245350719,NA,19 +"41","Atlantis","10",51.5980309799267,2.78036776500414,"NA ","NA ",7,209.190777628442,NA,19 +"531","Atlantis","10",51.5719121443596,2.75689270521649,"NA ","NA ",7,185.30364920127,NA,19 +"67","Atlantis","10",51.5414295826333,2.75234292136182,"NA ","NA ",7,130.685666893619,NA,19 +"832","Atlantis","10",51.5237199068605,2.78544274922618,"NA ","NA ",7,74.6324525672839,NA,19 +"945","Atlantis","10",51.5324959397904,2.83683991947561,"NA ","NA ",7,58.674026304408,NA,19 +"64","Atlantis","10",51.561529464133,2.90705724899312,"NA ","NA ",7,49.2131200311583,NA,20 +"121","Atlantis","10",51.5746588334007,2.93154744419966,"NA ","NA ",7,17.6201584184309,NA,20 +"301","Atlantis","10",51.5929334138044,2.9408903583399,"NA ","NA ",7,264.515571026057,NA,20 +"451","Atlantis","10",51.590746743699,2.90432815237604,"NA ","NA ",7,250.784756822194,NA,20 +"56","Atlantis","10",51.5834985670001,2.87088212154407,"NA ","NA ",7,238.600304569039,NA,20 +"706","Atlantis","10",51.5722806199788,2.84132252267826,"NA ","NA ",7,120.553867626117,NA,20 +"891","Atlantis","10",51.5631708315536,2.8661398436365,"NA ","NA ",7,88.156907708075,NA,20 +"952","Atlantis","10",51.5638105371982,2.8983390973349,"NA ","NA ",7,84.2180197009841,NA,20 +"51","Atlantis","10",51.5685438325653,2.96820818446556,"NA ","NA ",7,83.9097403780434,NA,21 +"11","Atlantis","10",51.570876984508,3.00346952782025,"NA ","NA ",7,86.0179104216133,NA,21 +"25","Atlantis","10",51.5724014223697,3.03882634334442,"NA ","NA ",7,266.550026704853,NA,21 +"48","Atlantis","10",51.5712348141474,3.00780136419531,"NA ","NA ",7,277.990039006788,NA,21 +"627","Atlantis","10",51.5742401757193,2.97329241203134,"NA ","NA ",7,327.639097160955,NA,21 +"80","Atlantis","10",51.5893708578177,2.95785946049412,"NA ","NA ",7,40.4326495169069,NA,21 +"901","Atlantis","10",51.6059022907444,2.98054214135047,"NA ","NA ",7,53.3460368523662,NA,21 +"962","Atlantis","10",51.6193421730519,3.00963954100231,"NA ","NA ",7,57.6831813409992,NA,21 +"65","Atlantis","10",51.643224403163,3.06964218956737,"NA ","NA ",6.5,51.2932161950055,NA,22 +"122","Atlantis","10",51.6569511607697,3.09726074063986,"NA ","NA ",6.5,36.9419981346725,NA,22 +"227","Atlantis","10",51.6747084921813,3.11879578368602,"NA ","NA ",6.5,330.522525622203,NA,22 +"401","Atlantis","10",51.691667364506,3.10333079211022,"NA ","NA ",6.5,271.962932458227,NA,22 +"55","Atlantis","10",51.6924614500194,3.06567033450651,"NA ","NA ",6.5,242.931743982912,NA,22 +"707","Atlantis","10",51.6825485730974,3.03439732578238,"NA ","NA ",6.5,172.855931872011,NA,22 +"862","Atlantis","10",51.6616720295974,3.03861559539397,"NA ","NA ",6.5,130.086015973804,NA,22 +"946","Atlantis","10",51.6477374150869,3.06529188788956,"NA ","NA ",6.5,119.142378882034,NA,22 +"91","Atlantis","10",51.6222961403463,3.14522105459243,"NA ","NA ",6.5,116.340029486186,NA,23 +"251","Atlantis","10",51.608749715665,3.18925180321037,"NA ","NA ",6.5,262.098525869647,NA,23 +"532","Atlantis","10",51.6048968464811,3.14465212507781,"NA ","NA ",6.5,263.41867607954,NA,23 +"641","Atlantis","10",51.601413009022,3.09617646974437,"NA ","NA ",6.5,257.057751810839,NA,23 +"735","Atlantis","10",51.5944830037437,3.04770378375592,"NA ","NA ",6.5,250.285042430043,NA,23 +"811","Atlantis","10",51.5838069407516,2.9998009113493,"NA ","NA ",6.5,243.492468396902,NA,23 +"881","Atlantis","10",51.5700299513879,2.95538593901028,"NA ","NA ",6.5,237.146351447182,NA,23 +"947","Atlantis","10",51.5543095078194,2.91625075021695,"NA ","NA ",6.5,231.050503346659,NA,23 +"10","Atlantis","10",51.5064241571048,2.82791261704117,"NA ","NA ",10.5,228.052965521571,NA,24 +"27","Atlantis","10",51.4775742458891,2.77639196676881,"NA ","NA ",10.5,272.405910755895,NA,24 +"57","Atlantis","10",51.4792311901461,2.71240557968916,"NA ","NA ",10.5,286.209895564253,NA,24 +"711","Atlantis","10",51.4913813735742,2.64517430470052,"NA ","NA ",10.5,280.996503120313,NA,24 +"796","Atlantis","10",51.4987522340351,2.58411117908323,"NA ","NA ",10.5,277.468288720832,NA,24 +"863","Atlantis","10",51.5045178794804,2.51318912261602,"NA ","NA ",10.5,274.9788959083,NA,24 +"911","Atlantis","10",51.5078753183932,2.45096597488697,"NA ","NA ",10.5,273.130473465621,NA,24 +"963","Atlantis","10",51.5103365345573,2.37799419055167,"NA ","NA ",10.5,271.636337654046,NA,24 +"58","Atlantis","10",51.5126809773197,2.24365688887805,"NA ","NA ",10,273.031998812067,NA,25 +"101","Atlantis","10",51.5151589836812,2.16773649986627,"NA ","NA ",10,274.631632243042,NA,25 +"152","Atlantis","10",51.5185140701564,2.10080512022502,"NA ","NA ",10,276.567951220763,NA,25 +"228","Atlantis","10",51.5241896636715,2.02120145291886,"NA ","NA ",10,278.774243672067,NA,25 +"29","Atlantis","10",51.5303613383316,1.95674251426496,"NA ","NA ",10,281.049697913105,NA,25 +"402","Atlantis","10",51.5393460325274,1.88257763553817,"NA ","NA ",10,277.026582381738,NA,25 +"628","Atlantis","10",51.5446318612083,1.8133542621132,"NA ","NA ",10,177.140680938151,NA,25 +"892","Atlantis","10",51.5004881782828,1.81689606028183,"NA ","NA ",10,170.209519897451,NA,25 +"810","Atlantis","10",51.4280268201111,1.83379973982865,"NA ","NA ",6.5,178.730060760429,NA,26 +"161","Atlantis","10",51.4010288246967,1.83475907830565,"NA ","NA ",6.5,187.771013378886,NA,26 +"291","Atlantis","10",51.3725816358339,1.82854029598571,"NA ","NA ",6.5,278.2584527646,NA,26 +"591","Atlantis","10",51.3746659839849,1.80550993152737,"NA ","NA ",6.5,359.506947159512,NA,26 +"723","Atlantis","10",51.4024674628562,1.80512642559049,"NA ","NA ",6.5,9.52473552958037,NA,26 +"812","Atlantis","10",51.4304355910934,1.81265326014516,"NA ","NA ",6.5,15.3540776641109,NA,26 +"882","Atlantis","10",51.4568672161642,1.82430116813057,"NA ","NA ",6.5,19.9367369125174,NA,26 +"953","Atlantis","10",51.4872160849042,1.8419803443092,"NA ","NA ",6.5,23.5597985859206,NA,26 +"510","Atlantis","10",51.5295680763202,1.8725223005238,"NA ","NA ",7,24.8065008226824,NA,27 +"102","Atlantis","10",51.548372933816,1.88649992440721,"NA ","NA ",7,24.5105125959321,NA,27 +"162","Atlantis","10",51.5655297996034,1.89908470871133,"NA ","NA ",7,23.4745936940272,NA,27 +"271","Atlantis","10",51.5844850211733,1.91233354452743,"NA ","NA ",7,0.153170433558898,NA,27 +"741","Atlantis","10",51.6031290601031,1.91241379143893,"NA ","NA ",7,15.0321068741245,NA,27 +"851","Atlantis","10",51.6231444350804,1.92107188145305,"NA ","NA ",7,17.9502250778125,NA,27 +"912","Atlantis","10",51.6410490579733,1.93041867890625,"NA ","NA ",7,19.0787815269824,NA,27 +"964","Atlantis","10",51.6605684389254,1.94130222646544,"NA ","NA ",7,19.7509468134978,NA,27 +"712","Atlantis","10",51.7073910746776,1.96506452563161,"NA ","NA ",7,2.83098137036984,NA,28 +"153","Atlantis","10",51.7355259548106,1.96731108392378,"NA ","NA ",7,333.402622284869,NA,28 +"272","Atlantis","10",51.7608598222917,1.94681540876639,"NA ","NA ",7,287.341623927529,NA,28 +"411","Atlantis","10",51.7694837343615,1.90214277976068,"NA ","NA ",7,255.672216275534,NA,28 +"548","Atlantis","10",51.7624520149289,1.85771406768505,"NA ","NA ",7,226.192780542173,NA,28 +"69","Atlantis","10",51.7429723867626,1.82492400297116,"NA ","NA ",7,161.477229921513,NA,28 +"852","Atlantis","10",51.7169738969013,1.83898296683347,"NA ","NA ",7,117.944186499417,NA,28 +"948","Atlantis","10",51.7033643775959,1.88035941017702,"NA ","NA ",7,105.440416894741,NA,28 +"713","Atlantis","10",51.6896841233515,1.97116993792232,"NA ","NA ",7,102.279863462279,NA,29 +"163","Atlantis","10",51.6829032465524,2.02133568628984,"NA ","NA ",7,120.090860459457,NA,29 +"412","Atlantis","10",51.668825647802,2.06048853791972,"NA ","NA ",7,233.162089138367,NA,29 +"601","Atlantis","10",51.6500392190108,2.02008569553024,"NA ","NA ",7,231.773096648065,NA,29 +"714","Atlantis","10",51.6307405664498,1.98062852968804,"NA ","NA ",7,225.377540753438,NA,29 +"801","Atlantis","10",51.6083339807478,1.94407965297389,"NA ","NA ",7,218.93684986066,NA,29 +"872","Atlantis","10",51.5859181703062,1.9149359192155,"NA ","NA ",7,212.705274331819,NA,29 +"949","Atlantis","10",51.5584879669015,1.88660904693802,"NA ","NA ",7,206.734411813249,NA,29 +"715","Atlantis","10",51.5016285178081,1.84408316273313,"NA ","NA ",6.5,201.500357559591,NA,30 +"131","Atlantis","10",51.4752678124626,1.82741230760266,"NA ","NA ",6.5,199.588580222793,NA,30 +"20","Atlantis","10",51.447908663751,1.81179093998333,"NA ","NA ",6.5,197.193504606517,NA,30 +"283","Atlantis","10",51.4213751138301,1.79862522606383,"NA ","NA ",6.5,193.579488275051,NA,30 +"397","Atlantis","10",51.3936910483812,1.78790849503275,"NA ","NA ",6.5,166.239634780486,NA,30 +"708","Atlantis","10",51.3771328187051,1.79440474540826,"NA ","NA ",6.5,20.3220679525574,NA,30 +"853","Atlantis","10",51.4057048449381,1.81136859377239,"NA ","NA ",6.5,15.1703127114286,NA,30 +"931","Atlantis","10",51.4317127817975,1.82267970699402,"NA ","NA ",6.5,12.8947333064943,NA,30 +"716","Atlantis","10",51.4863868411377,1.84191753283998,"NA ","NA ",6,11.0384036788864,NA,31 +"154","Atlantis","10",51.5134931726988,1.85041439401823,"NA ","NA ",6,9.45843947817315,NA,31 +"273","Atlantis","10",51.540213314291,1.85757159829634,"NA ","NA ",6,243.903797482368,NA,31 +"592","Atlantis","10",51.5377869887405,1.84960851078705,"NA ","NA ",6,194.913119139316,NA,31 +"717","Atlantis","10",51.5125172541891,1.83879483898096,"NA ","NA ",6,193.355377662074,NA,31 +"802","Atlantis","10",51.4859530864001,1.82866725025039,"NA ","NA ",6,192.441019587379,NA,31 +"883","Atlantis","10",51.4572993677305,1.81852222700785,"NA ","NA ",6,191.786242869848,NA,31 +"9410","Atlantis","10",51.4329091173024,1.81035898022218,"NA ","NA ",6,191.258446083142,NA,31 +"813","Atlantis","10",51.3765147896008,1.79275099767948,"NA ","NA ",6.5,190.29695563458,NA,32 +"164","Atlantis","10",51.3493566967484,1.78485128783267,"NA ","NA ",6.5,189.160748695827,NA,32 +"284","Atlantis","10",51.3218460908272,1.77775246791538,"NA ","NA ",6.5,60.0530578139242,NA,32 +"611","Atlantis","10",51.3244854304961,1.78508419016465,"NA ","NA ",6.5,14.2891543443665,NA,32 +"736","Atlantis","10",51.3525289136782,1.79652113991628,"NA ","NA ",6.5,13.2669220665479,NA,32 +"814","Atlantis","10",51.3788673510735,1.80647068975065,"NA ","NA ",6.5,12.7583890827931,NA,32 +"884","Atlantis","10",51.4068013423707,1.81661071685754,"NA ","NA ",6.5,12.3801628827924,NA,32 +"954","Atlantis","10",51.4392002069608,1.82801967612739,"NA ","NA ",6.5,12.1035418914246,NA,32 +"97","Atlantis","10",51.5070117427446,1.84462191459954,"NA ","NA ",10,356.036720864574,NA,33 +"18","Atlantis","10",51.5509169638078,1.83973000211014,"NA ","NA ",10,346.981756756757,NA,33 +"285","Atlantis","10",51.5941738378369,1.82363058802579,"NA ","NA ",10,340.277150628286,NA,33 +"403","Atlantis","10",51.6361839919628,1.79936321379491,"NA ","NA ",10,345.69362792195,NA,33 +"642","Atlantis","10",51.6770349599872,1.78256264449984,"NA ","NA ",10,114.476489165787,NA,33 +"823","Atlantis","10",51.6588316230245,1.84695989931848,"NA ","NA ",10,122.160163036744,NA,33 +"902","Atlantis","10",51.634483664956,1.90930696510068,"NA ","NA ",10,122.861306134005,NA,33 +"955","Atlantis","10",51.6136393794617,1.96124319487951,"NA ","NA ",10,122.980608278902,NA,33 +"511","Atlantis","10",51.5674750754267,2.07651418051539,"NA ","NA ",9.5,121.641984287165,NA,34 +"98","Atlantis","10",51.5495210502349,2.12334449056771,"NA ","NA ",9.5,120.475707840481,NA,34 +"155","Atlantis","10",51.5273561195441,2.18384309126721,"NA ","NA ",9.5,119.042037105188,NA,34 +"213","Atlantis","10",51.5102631840408,2.23327391053233,"NA ","NA ",9.5,117.371471596835,NA,34 +"302","Atlantis","10",51.4925407248981,2.28821463973597,"NA ","NA ",9.5,120.559881646775,NA,34 +"551","Atlantis","10",51.4740537736739,2.3384524350749,"NA ","NA ",9.5,248.875404206697,NA,34 +"797","Atlantis","10",51.4605113752114,2.28225124451801,"NA ","NA ",9.5,238.337456455348,NA,34 +"913","Atlantis","10",51.4404771474049,2.23016472850862,"NA ","NA ",9.5,227.580294468774,NA,34 +"815","Atlantis","10",51.4180323501106,2.19660553765787,"NA ","NA ",8,189.41699236304,NA,36 +"19","Atlantis","10",51.384840106768,2.18778464427209,"NA ","NA ",8,138.895643922157,NA,36 +"341","Atlantis","10",51.3587662053165,2.22420801461536,"NA ","NA ",8,100.216278370969,NA,36 +"479","Atlantis","10",51.3526878055246,2.27810298083167,"NA ","NA ",8,76.3703883337867,NA,36 +"602","Atlantis","10",51.3605915058677,2.33038252784589,"NA ","NA ",8,37.5343961178062,NA,36 +"751","Atlantis","10",51.3853992114164,2.36092730168449,"NA ","NA ",8,338.441946242202,NA,36 +"873","Atlantis","10",51.4160785635613,2.3414912782,"NA ","NA ",8,312.874426435515,NA,36 +"956","Atlantis","10",51.441332843913,2.29783777957305,"NA ","NA ",8,303.662082206894,NA,36 +"66","Atlantis","10",51.4759765217575,2.21214919761337,"NA ","NA ",7.5,305.559626343029,NA,37 +"111","Atlantis","10",51.4907463252173,2.17895585143813,"NA ","NA ",7.5,311.991423923171,NA,37 +"201","Atlantis","10",51.5116833524365,2.14157120236624,"NA ","NA ",7.5,358.970311292679,NA,37 +"431","Atlantis","10",51.5338276429146,2.14093137199988,"NA ","NA ",7.5,111.395442212533,NA,37 +"603","Atlantis","10",51.5232187976306,2.18441661515379,"NA ","NA ",7.5,138.867158158356,NA,37 +"737","Atlantis","10",51.5008744171225,2.21575971321401,"NA ","NA ",7.5,166.167094547711,NA,37 +"841","Atlantis","10",51.4729831818649,2.22678521450395,"NA ","NA ",7.5,191.507326185191,NA,37 +"932","Atlantis","10",51.4443936614806,2.21744688800087,"NA ","NA ",7.5,208.852775111926,NA,37 +"156","Atlantis","10",51.3944033966772,2.14496155131861,"NA ","NA ",8.5,331.012582938542,NA,38 +"33","Atlantis","10",51.4281460307497,2.11497532662286,"NA ","NA ",8.5,2.29462981666478,NA,38 +"452","Atlantis","10",51.4718490793411,2.11778668761162,"NA ","NA ",8.5,15.4322475013917,NA,38 +"552","Atlantis","10",51.5104261766116,2.13489800548502,"NA ","NA ",8.5,31.661080279639,NA,38 +"661","Atlantis","10",51.5457690771276,2.16994972325282,"NA ","NA ",8.5,62.5145185257529,NA,38 +"781","Atlantis","10",51.5646709039593,2.22844181256728,"NA ","NA ",8.5,102.19113665633,NA,38 +"885","Atlantis","10",51.55584405615,2.29401752367914,"NA ","NA ",8.5,124.543946406696,NA,38 +"957","Atlantis","10",51.532134288708,2.34935227041458,"NA ","NA ",8.5,134.736103720357,NA,38 +"718","Atlantis","10",51.472058625717,2.38944013565276,"NA ","NA ",6.5,227.715413665541,NA,40 +"132","Atlantis","10",51.4514707455214,2.35312326686487,"NA ","NA ",6.5,226.903998081864,NA,40 +"202","Atlantis","10",51.428776879462,2.31423278637551,"NA ","NA ",6.5,226.094784278899,NA,40 +"286","Atlantis","10",51.4050967211771,2.27480416832133,"NA ","NA ",6.5,225.382272354331,NA,40 +"36","Atlantis","10",51.3844161084897,2.24122969193625,"NA ","NA ",6.5,224.810522935505,NA,40 +"481","Atlantis","10",51.3603740148462,2.20299033200309,"NA ","NA ",6.5,232.148243479758,NA,40 +"798","Atlantis","10",51.3507314503012,2.18312679778577,"NA ","NA ",6.5,37.814942119664,NA,40 +"921","Atlantis","10",51.3756587082663,2.21412454214744,"NA ","NA ",6.5,38.0331024864859,NA,40 +"719","Atlantis","10",51.4226698481688,2.27423294459391,"NA ","NA ",5.5,41.5607553228106,NA,41 +"133","Atlantis","10",51.4421513571541,2.30194900048958,"NA ","NA ",5.5,44.0250320253978,NA,41 +"203","Atlantis","10",51.462388378349,2.33335016176294,"NA ","NA ",5.5,46.8223065890246,NA,41 +"287","Atlantis","10",51.4821362321189,2.36715285655189,"NA ","NA ",5.5,50.0850234409862,NA,41 +"371","Atlantis","10",51.4999047131938,2.40128129885039,"NA ","NA ",5.5,54.5130000063502,NA,41 +"482","Atlantis","10",51.5149944047186,2.4353029779664,"NA ","NA ",5.5,84.3298676931901,NA,41 +"782","Atlantis","10",51.5167816778641,2.46428820479236,"NA ","NA ",5.5,222.715556012168,NA,41 +"922","Atlantis","10",51.4966259850946,2.4344023391792,"NA ","NA ",5.5,228.652089874794,NA,41 +"816","Atlantis","10",51.4617366596717,2.3699026525208,"NA ","NA ",6,228.191231465126,NA,42 +"165","Atlantis","10",51.444139624114,2.33834300174198,"NA ","NA ",6,226.754498180399,NA,42 +"252","Atlantis","10",51.4272316327209,2.30951772432954,"NA ","NA ",6,224.517805695663,NA,42 +"398","Atlantis","10",51.4083937728392,2.27982748333399,"NA ","NA ",6,70.3135399652381,NA,42 +"7110","Atlantis","10",51.4106797889781,2.29007320008426,"NA ","NA ",6,45.8633589016054,NA,42 +"824","Atlantis","10",51.4297742678844,2.32164334487306,"NA ","NA ",6,44.5186757469554,NA,42 +"893","Atlantis","10",51.4476759592716,2.34989420024197,"NA ","NA ",6,43.7671985646859,NA,42 +"958","Atlantis","10",51.4668397556512,2.37936589769827,"NA ","NA ",6,43.2183243848614,NA,42 +"610","Atlantis","10",51.5050116193871,2.43638158498776,"NA ","NA ",6.5,41.7537813438715,NA,43 +"134","Atlantis","10",51.5295149477312,2.47154836289664,"NA ","NA ",6.5,40.8377855768706,NA,43 +"204","Atlantis","10",51.5508082235661,2.50115112328328,"NA ","NA ",6.5,40.0368623691525,NA,43 +"288","Atlantis","10",51.5709899914498,2.52843687654668,"NA ","NA ",6.5,39.5859157832857,NA,43 +"413","Atlantis","10",51.5936914703504,2.55865754151715,"NA ","NA ",6.5,128.687330881643,NA,43 +"738","Atlantis","10",51.5898906537141,2.56629659202984,"NA ","NA ",6.5,204.062056836278,NA,43 +"854","Atlantis","10",51.5640430760691,2.5477311853801,"NA ","NA ",6.5,203.876966596862,NA,43 +"933","Atlantis","10",51.5387087388032,2.5297020934313,"NA ","NA ",6.5,203.315448642125,NA,43 +"720","Atlantis","10",51.4880852973231,2.49309985723312,"NA ","NA ",6,211.801036330638,NA,44 +"144","Atlantis","10",51.4685695081542,2.47367628402924,"NA ","NA ",6,224.037501190231,NA,44 +"24","Atlantis","10",51.4517055346228,2.44751364355803,"NA ","NA ",6,249.298060201768,NA,44 +"399","Atlantis","10",51.4432355230623,2.41157785664811,"NA ","NA ",6,271.409875206779,NA,44 +"571","Atlantis","10",51.4438165837391,2.37329307461176,"NA ","NA ",6,248.402286242412,NA,44 +"752","Atlantis","10",51.435537675661,2.33976617181468,"NA ","NA ",6,210.44483643908,NA,44 +"874","Atlantis","10",51.4143344375616,2.31978621347575,"NA ","NA ",6,197.699103975404,NA,44 +"9411","Atlantis","10",51.3931067925342,2.30892985748516,"NA ","NA ",6,192.680731299323,NA,44 +"103","Atlantis","10",51.3383793064278,2.29582225432618,"NA ","NA ",6.5,94.1362256785021,NA,45 +"292","Atlantis","10",51.3368421350111,2.32973788315944,"NA ","NA ",6.5,39.8538890282331,NA,45 +"404","Atlantis","10",51.3618922117257,2.36323387055661,"NA ","NA ",6.5,32.484675649007,NA,45 +"483","Atlantis","10",51.3864251006581,2.38826596571341,"NA ","NA ",6.5,27.5810917693445,NA,45 +"561","Atlantis","10",51.4129048668997,2.41044518012127,"NA ","NA ",6.5,21.3020792966611,NA,45 +"651","Atlantis","10",51.4412911119116,2.42820349191706,"NA ","NA ",6.5,4.41322174379633,NA,45 +"783","Atlantis","10",51.4713016548063,2.43192179653092,"NA ","NA ",6.5,277.0555323802,NA,45 +"934","Atlantis","10",51.4746308307183,2.38863318900419,"NA ","NA ",6.5,240.465692423157,NA,45 +"104","Atlantis","10",51.4821250278197,2.41545223031491,"NA ","NA ",7,134.854713258362,NA,47 +"261","Atlantis","10",51.4624956298084,2.447112051032,"NA ","NA ",7,188.488278409848,NA,47 +"361","Atlantis","10",51.4292172379851,2.43914630942196,"NA ","NA ",7,196.690733405886,NA,47 +"441","Atlantis","10",51.396349684701,2.4233518525513,"NA ","NA ",7,199.880247195624,NA,47 +"521","Atlantis","10",51.3615959581001,2.40322623883568,"NA ","NA ",7,201.27588024243,NA,47 +"604","Atlantis","10",51.3283389462593,2.38250223720396,"NA ","NA ",7,201.097825406765,NA,47 +"691","Atlantis","10",51.2969259418883,2.36312020956264,"NA ","NA ",7,175.741057419796,NA,47 +"914","Atlantis","10",51.27617848263,2.36559005935495,"NA ","NA ",7,39.2729258794342,NA,47 +"512","Atlantis","10",51.3193619238452,2.41996975960463,"NA ","NA ",7,34.733302805001,NA,48 +"112","Atlantis","10",51.3398133870234,2.44266925387578,"NA ","NA ",7,32.0812611800872,NA,48 +"181","Atlantis","10",51.3578969359356,2.46082340637496,"NA ","NA ",7,27.0679811474099,NA,48 +"303","Atlantis","10",51.3767949241109,2.47629576763276,"NA ","NA ",7,78.1350163929159,NA,48 +"724","Atlantis","10",51.3800889071654,2.50143607976154,"NA ","NA ",7,88.7184573797165,NA,48 +"833","Atlantis","10",51.3805562336264,2.53525437761884,"NA ","NA ",7,81.0837712521643,NA,48 +"903","Atlantis","10",51.3839123175014,2.56958198004137,"NA ","NA ",7,77.162747326071,NA,48 +"965","Atlantis","10",51.3894609886777,2.60864746530639,"NA ","NA ",7,74.8062647391097,NA,48 +"817","Atlantis","10",51.4079539055665,2.69119434067207,"NA ","NA ",9,48.7339187394151,NA,49 +"182","Atlantis","10",51.4335614837814,2.73802291302778,"NA ","NA ",9,25.5523345246289,NA,49 +"289","Atlantis","10",51.4637702868879,2.76120689380943,"NA ","NA ",9,7.06748424006969,NA,49 +"405","Atlantis","10",51.5007126454301,2.76856451202065,"NA ","NA ",9,357.048800245981,NA,49 +"549","Atlantis","10",51.536417962161,2.76560519635155,"NA ","NA ",9,60.3472932140784,NA,49 +"803","Atlantis","10",51.5471500162593,2.79593032089495,"NA ","NA ",9,131.981194931317,NA,49 +"894","Atlantis","10",51.5241983815326,2.83691380116647,"NA ","NA ",9,134.898033831465,NA,49 +"959","Atlantis","10",51.4989275450223,2.87764098537757,"NA ","NA ",9,135.759632977445,NA,49 +"725","Atlantis","10",51.4371790015531,2.97340715100944,"NA ","NA ",11,136.20823361154,NA,50 +"145","Atlantis","10",51.4058529245249,3.02153563578368,"NA ","NA ",11,134.962225321457,NA,50 +"241","Atlantis","10",51.3748619592689,3.0712315918375,"NA ","NA ",11,120.018654977745,NA,50 +"453","Atlantis","10",51.3523884938481,3.13346634509528,"NA ","NA ",11,66.6300157835427,NA,50 +"671","Atlantis","10",51.3703153775197,3.19999016192468,"NA ","NA ",11,65.3952153018722,NA,50 +"799","Atlantis","10",51.3888291895018,3.2648391289863,"NA ","NA ",11,71.9482841720463,NA,50 +"886","Atlantis","10",51.4034472235107,3.33684494886645,"NA ","NA ",11,76.7635955285895,NA,50 +"9412","Atlantis","10",51.4125505985908,3.39900814204346,"NA ","NA ",11,80.3441517028733,NA,50 +"4","Atlantis","10",51.4239997965828,3.51351829931597,"NA ","NA ",5.5,81.6375589074348,NA,51 +"726","Atlantis","10",51.4272026564517,3.54852023281246,"NA ","NA ",5.5,81.2977017933609,NA,51 +"113","Atlantis","10",51.4310665607989,3.58908395190189,"NA ","NA ",5.5,80.740538105767,NA,51 +"166","Atlantis","10",51.4352781809406,3.63059575719494,"NA ","NA ",5.5,79.6514551159464,NA,51 +"232","Atlantis","10",51.4401078839989,3.67309444601828,"NA ","NA ",5.5,75.5778463820828,NA,51 +"362","Atlantis","10",51.446199336868,3.71113790083009,"NA ","NA ",5.5,281.168651674137,NA,51 +"631","Atlantis","10",51.4502015354441,3.67857403540263,"NA ","NA ",5.5,268.876960834995,NA,51 +"771","Atlantis","10",51.4496982641763,3.63795336865387,"NA ","NA ",5.5,267.445540069597,NA,51 +"229","Atlantis","10",51.4483794950425,3.59752281888496,"NA ","NA ",0,250.894677391241,NA,52 +"317","Atlantis","10",51.4481859445331,3.59662624805429,"NA ","NA ",0,250.894771667926,NA,52 +"3910","Atlantis","10",51.4479828378931,3.59568541101147,"NA ","NA ",0,250.894884486292,NA,52 +"4710","Atlantis","10",51.4477627801093,3.59466605234063,"NA ","NA ",0,250.894924020973,NA,52 +"5410","Atlantis","10",51.4475655944702,3.59375264274599,"NA ","NA ",0,250.895046410921,NA,52 +"629","Atlantis","10",51.4473455366864,3.59273328407515,"NA ","NA ",0,250.895097817619,NA,52 +"709","Atlantis","10",51.4471424300464,3.59179244703233,"NA ","NA ",0,250.895157473936,NA,52 +"7910","Atlantis","10",51.446948879537,3.59089587620166,"NA ","NA ",0,250.895277269195,NA,52 +"2210","Atlantis","10",51.4467905584682,3.58972583810299,"NA ","NA ",0,298.459136926807,NA,53 +"318","Atlantis","10",51.4468421331811,3.58957316708526,"NA ","NA ",0,298.459167653214,NA,53 +"3911","Atlantis","10",51.446896254282,3.58941295827193,"NA ","NA ",0,298.459202523917,NA,53 +"4711","Atlantis","10",51.4469548922937,3.58923937853858,"NA ","NA ",0,298.459224687363,NA,53 +"5411","Atlantis","10",51.4470074356467,3.58908384016097,"NA ","NA ",0,298.459260965736,NA,53 +"6210","Atlantis","10",51.4470660736584,3.58891026042762,"NA ","NA ",0,298.459285373773,NA,53 +"7010","Atlantis","10",51.4471201947593,3.58875005161429,"NA ","NA ",0,298.459310214695,NA,53 +"7911","Atlantis","10",51.4471717694722,3.58859738059656,"NA ","NA ",0,298.459345760342,NA,53 +"2211","Atlantis","10",51.4472179667061,3.5885894618987,"NA ","NA ",0,96.3815947352728,NA,54 +"319","Atlantis","10",51.4472078197648,3.5887350317552,"NA ","NA ",0,96.3815904878781,NA,54 +"3912","Atlantis","10",51.4471971718405,3.58888778880309,"NA ","NA ",0,96.3815839599819,NA,54 +"4712","Atlantis","10",51.4471856352476,3.58905329485165,"NA ","NA ",0,96.3815891685895,NA,54 +"5412","Atlantis","10",51.4471752977335,3.589201598699,"NA ","NA ",0,96.3815809281225,NA,54 +"6211","Atlantis","10",51.4471637611406,3.58936710474756,"NA ","NA ",0,96.3815843772966,NA,54 +"7011","Atlantis","10",51.4471531132163,3.58951986179545,"NA ","NA ",0,96.381585748181,NA,54 +"7912","Atlantis","10",51.447142966275,3.58966543165195,"NA ","NA ",0,96.3815777070517,NA,54 +"2212","Atlantis","10",51.4435099622133,3.58393199587911,"NA ","NA ",0,157.563334668848,NA,57 +"3110","Atlantis","10",51.4432858363217,3.58408047567475,"NA ","NA ",0,157.563230158233,NA,57 +"3913","Atlantis","10",51.4430506447073,3.58423628633362,"NA ","NA ",0,157.563116635748,NA,57 +"4713","Atlantis","10",51.4427958241586,3.58440510084548,"NA ","NA ",0,157.563016575363,NA,57 +"5413","Atlantis","10",51.4425674888919,3.58455636928442,"NA ","NA ",0,157.562902796929,NA,57 +"6212","Atlantis","10",51.4423126683432,3.58472518379628,"NA ","NA ",0,157.562799444905,NA,57 +"7012","Atlantis","10",51.4420774767288,3.58488099445515,"NA ","NA ",0,157.562700669451,NA,57 +"7913","Atlantis","10",51.4418533508371,3.58502947425079,"NA ","NA ",0,157.562589082007,NA,57 +"2213","Atlantis","10",51.4422610911189,3.58642801686979,"NA ","NA ",0,49.443521857173,NA,58 +"3111","Atlantis","10",51.4428508616569,3.58753369233993,"NA ","NA ",0,49.4431259644638,NA,58 +"3914","Atlantis","10",51.4434697508176,3.58869395810375,"NA ","NA ",0,49.4426885136104,NA,58 +"4714","Atlantis","10",51.4441402920523,3.58995105886933,"NA ","NA ",0,49.4423455936371,NA,58 +"5414","Atlantis","10",51.4447411392465,3.59107750036,"NA ","NA ",0,49.4419004884869,NA,58 +"6213","Atlantis","10",51.4454116804812,3.59233460112558,"NA ","NA ",0,49.4415387328444,NA,58 +"7013","Atlantis","10",51.4460305696419,3.5934948668894,"NA ","NA ",0,49.4411854829623,NA,58 +"7914","Atlantis","10",51.4466203401799,3.59460054235954,"NA ","NA ",0,49.4407490860348,NA,58 +"2214","Atlantis","10",51.446966755835,3.59526987679553,"NA ","NA ",0,228.457466492451,NA,59 +"3112","Atlantis","10",51.4466876365506,3.59476443280641,"NA ","NA ",0,228.457654380932,NA,59 +"3915","Atlantis","10",51.4463947363657,3.59423403363237,"NA ","NA ",0,228.457861724642,NA,59 +"4715","Atlantis","10",51.446077390928,3.59365936769934,"NA ","NA ",0,228.458025743782,NA,59 +"5415","Atlantis","10",51.4457930294213,3.59314443081601,"NA ","NA ",0,228.458236469224,NA,59 +"6214","Atlantis","10",51.4454756839836,3.59256976488298,"NA ","NA ",0,228.458409188789,NA,59 +"7014","Atlantis","10",51.4451827837987,3.59203936570895,"NA ","NA ",0,228.458577553899,NA,59 +"7915","Atlantis","10",51.4449036645143,3.59153392171982,"NA ","NA ",0,228.458784141525,NA,59 +"2215","Atlantis","10",51.4444044665842,3.59143616114104,"NA ","NA ",0,122.913373999373,NA,60 +"3113","Atlantis","10",51.4442359257799,3.59185393748028,"NA ","NA ",0,122.913267096819,NA,60 +"3916","Atlantis","10",51.4440590636442,3.5922923406071,"NA ","NA ",0,122.913147343124,NA,60 +"4716","Atlantis","10",51.4438674407134,3.59276733256458,"NA ","NA ",0,122.913062676137,NA,60 +"5416","Atlantis","10",51.4436957344941,3.59319295528577,"NA ","NA ",0,122.912939407899,NA,60 +"6215","Atlantis","10",51.4435041115633,3.59366794724325,"NA ","NA ",0,122.912848269823,NA,60 +"7015","Atlantis","10",51.4433272494276,3.59410635037007,"NA ","NA ",0,122.912757498105,NA,60 +"7916","Atlantis","10",51.4431587086233,3.59452412670931,"NA ","NA ",0,122.912636687506,NA,60 +"2216","Atlantis","10",51.4431924865193,3.59536051706544,"NA ","NA ",0,46.1662995518181,NA,61 +"3114","Atlantis","10",51.4433913179458,3.59569278105757,"NA ","NA ",0,46.166165431704,NA,61 +"3917","Atlantis","10",51.4435999662364,3.59604144985327,"NA ","NA ",0,46.1660178156264,NA,61 +"4717","Atlantis","10",51.4438260281734,3.5964192183126,"NA ","NA ",0,46.1658988339772,NA,61 +"5417","Atlantis","10",51.4440285939122,3.59675772265379,"NA ","NA ",0,46.1657491483574,NA,61 +"6216","Atlantis","10",51.4442546558492,3.59713549111312,"NA ","NA ",0,46.1656242863921,NA,61 +"7016","Atlantis","10",51.4444633041398,3.59748415990882,"NA ","NA ",0,46.1655029934375,NA,61 +"7917","Atlantis","10",51.4446621355663,3.59781642390095,"NA ","NA ",0,46.1653562279447,NA,61 +"2217","Atlantis","10",51.4195955455672,3.64041169549037,"NA ","NA ",7,134.187658398404,NA,62 +"321","Atlantis","10",51.3968505483152,3.67790532599695,"NA ","NA ",7,134.735699404868,NA,62 +"432","Atlantis","10",51.3709967692758,3.71969124148378,"NA ","NA ",7,136.404172962742,NA,62 +"572","Atlantis","10",51.3458444833875,3.75802306792708,"NA ","NA ",7,297.91730766727,NA,62 +"818","Atlantis","10",51.3558196523235,3.72786468593435,"NA ","NA ",7,309.558242347601,NA,62 +"887","Atlantis","10",51.3785495476706,3.68376379199623,"NA ","NA ",7,310.326011067856,NA,62 +"935","Atlantis","10",51.4018553653839,3.63973794502635,"NA ","NA ",7,310.703896609746,NA,62 +"971","Atlantis","10",51.4251660900882,3.59626519826118,"NA ","NA ",7,310.929082297984,NA,62 +"612","Atlantis","10",51.5891815333966,2.94128178036405,"NA ","NA ",7,68.1374052368741,NA,65 +"123","Atlantis","10",51.5982531400327,2.97770168428279,"NA ","NA ",7,78.6588805347247,NA,65 +"214","Atlantis","10",51.6035080626679,3.019946495369,"NA ","NA ",7,104.344745867596,NA,65 +"363","Atlantis","10",51.5970301485078,3.06068073275556,"NA ","NA ",7,178.864356709901,NA,65 +"613","Atlantis","10",51.575839024048,3.061356666671,"NA ","NA ",7,284.296891161518,NA,65 +"7918","Atlantis","10",51.5817872089156,3.02375666991709,"NA ","NA ",7,316.990010984704,NA,65 +"888","Atlantis","10",51.5996524362285,2.99692224476767,"NA ","NA ",7,328.001238407596,NA,65 +"9510","Atlantis","10",51.6228922746388,2.97353015965848,"NA ","NA ",7,333.949616596772,NA,65 +"819","Atlantis","10",51.6748344650293,2.93561570435061,"NA ","NA ",10,343.401814843195,NA,66 +"2218","Atlantis","10",51.7065660754229,2.92035191503996,"NA ","NA ",10,62.0904568156308,NA,66 +"573","Atlantis","10",51.7204978141085,2.96283200141993,"NA ","NA ",10,25.6917772338506,NA,66 +"739","Atlantis","10",51.7475788032988,2.98387665657736,"NA ","NA ",10,3.04345130651626,NA,66 +"825","Atlantis","10",51.7817145446038,2.98681032335519,"NA ","NA ",10,356.212524218767,NA,66 +"889","Atlantis","10",51.8147590438167,2.98327175871502,"NA ","NA ",10,352.86846656115,NA,66 +"936","Atlantis","10",51.8496577045236,2.97620328032459,"NA ","NA ",10,350.803189877398,NA,66 +"972","Atlantis","10",51.8829227846018,2.96747790045095,"NA ","NA ",10,349.496377775867,NA,66 +"42","Atlantis","10",51.939372245863,2.95013381695608,"NA ","NA ",10,349.120265212859,NA,67 +"99","Atlantis","10",51.9791306301822,2.93772725959237,"NA ","NA ",10,349.090068571778,NA,67 +"146","Atlantis","10",52.0106582390762,2.92785421234524,"NA ","NA ",10,348.736430944915,NA,67 +"205","Atlantis","10",52.0387437260901,2.9187609331082,"NA ","NA ",10,344.503620869858,NA,67 +"364","Atlantis","10",52.071703744403,2.90389349854416,"NA ","NA ",10,215.472908166206,NA,67 +"652","Atlantis","10",52.046374255513,2.87455041152793,"NA ","NA ",10,242.676410220228,NA,67 +"826","Atlantis","10",52.0318331714362,2.82883162816038,"NA ","NA ",10,286.617595758855,NA,67 +"937","Atlantis","10",52.041115034505,2.7782105108616,"NA ","NA ",10,304.405070810273,NA,67 +"614","Atlantis","10",52.0751646971522,2.70354573302441,"NA ","NA ",6.5,310.199169486487,NA,69 +"124","Atlantis","10",52.0909721224212,2.67309250393053,"NA ","NA ",6.5,310.963216764566,NA,69 +"191","Atlantis","10",52.1053580004234,2.64610817881859,"NA ","NA ",6.5,310.739868083491,NA,69 +"3115","Atlantis","10",52.1210531837869,2.61642391127131,"NA ","NA ",6.5,277.036010559781,NA,69 +"615","Atlantis","10",52.1236978005655,2.5814553887267,"NA ","NA ",6.5,241.15303913433,NA,69 +"784","Atlantis","10",52.1118850808412,2.54654984251756,"NA ","NA ",6.5,250.500957340176,NA,69 +"875","Atlantis","10",52.1044829878947,2.51253916170812,"NA ","NA ",6.5,256.344067484944,NA,69 +"9413","Atlantis","10",52.0989518708758,2.47551850930071,"NA ","NA ",6.5,260.392926998811,NA,69 +"105","Atlantis","10",52.0932461806969,2.38312897507042,"NA ","NA ",10,324.685281112762,NA,70 +"342","Atlantis","10",52.1140175622354,2.35916401905909,"NA ","NA ",10,66.070476512143,NA,70 +"581","Atlantis","10",52.1260787816451,2.40346671086319,"NA ","NA ",10,175.879174565559,NA,70 +"761","Atlantis","10",52.0954633215929,2.40705706344855,"NA ","NA ",10,203.245875718771,NA,70 +"842","Atlantis","10",52.0638689807206,2.38498354009768,"NA ","NA ",10,209.107860750483,NA,70 +"895","Atlantis","10",52.0371305851903,2.36078487563745,"NA ","NA ",10,211.966499123386,NA,70 +"938","Atlantis","10",52.0114510569811,2.33475143009522,"NA ","NA ",10,213.987312269769,NA,70 +"973","Atlantis","10",51.9816762378093,2.30216447395499,"NA ","NA ",10,215.453348981191,NA,70 +"513","Atlantis","10",51.9233240021511,2.23353118618281,"NA ","NA ",10,216.997978796066,NA,71 +"910","Atlantis","10",51.8941043351718,2.19786060199485,"NA ","NA ",10,217.930577686231,NA,71 +"157","Atlantis","10",51.8570663519005,2.15113676497312,"NA ","NA ",10,219.203793436985,NA,71 +"215","Atlantis","10",51.8273177306591,2.11188269983412,"NA ","NA ",10,220.90789564103,NA,71 +"293","Atlantis","10",51.7973187959803,2.06986338963121,"NA ","NA ",10,223.715716490585,NA,71 +"421","Atlantis","10",51.7666255522617,2.02245717696968,"NA ","NA ",10,222.933331654121,NA,71 +"681","Atlantis","10",51.7367035934267,1.97751812911909,"NA ","NA ",10,199.953744507403,NA,71 +"896","Atlantis","10",51.6989296228694,1.95539250796896,"NA ","NA ",10,195.957877471274,NA,71 +"616","Atlantis","10",51.6382877721035,1.92736840699354,"NA ","NA ",5.5,196.258058833734,NA,72 +"114","Atlantis","10",51.6209572333541,1.91922817377053,"NA ","NA ",5.5,196.579123461759,NA,72 +"192","Atlantis","10",51.5994423211306,1.90891641578977,"NA ","NA ",5.5,197.285880117088,NA,72 +"294","Atlantis","10",51.5812991199525,1.89983059034154,"NA ","NA ",5.5,199.850840124932,NA,72 +"522","Atlantis","10",51.5614004794338,1.88827521014208,"NA ","NA ",5.5,201.012704858698,NA,72 +"762","Atlantis","10",51.5419504055907,1.87626299138758,"NA ","NA ",5.5,198.382710138992,NA,72 +"876","Atlantis","10",51.5226726076176,1.86596693055465,"NA ","NA ",5.5,197.551673919932,NA,72 +"9414","Atlantis","10",51.5044510909145,1.85670812464676,"NA ","NA ",5.5,197.149146369752,NA,72 +"820","Atlantis","10",51.4595964378894,1.83389226907771,"NA ","NA ",6,200.02801911546,NA,73 +"167","Atlantis","10",51.4356510894815,1.81989090771064,"NA ","NA ",6,202.992293924786,NA,73 +"274","Atlantis","10",51.4104527910412,1.80274984879981,"NA ","NA ",6,211.844132973083,NA,73 +"49","Atlantis","10",51.3889653133723,1.78136543727986,"NA ","NA ",6,11.4024921579995,NA,73 +"7310","Atlantis","10",51.4138605228303,1.78941580349721,"NA ","NA ",6,21.0776115440332,NA,73 +"827","Atlantis","10",51.4373353904349,1.80393046190925,"NA ","NA ",6,23.2237832983826,NA,73 +"897","Atlantis","10",51.4612370370311,1.82039238083711,"NA ","NA ",6,24.5907871265154,NA,73 +"9511","Atlantis","10",51.4857737052652,1.83842588505898,"NA ","NA ",6,25.5836805747448,NA,73 +"727","Atlantis","10",51.5366234366396,1.87779387519567,"NA ","NA ",6.5,25.4719217542158,NA,74 +"135","Atlantis","10",51.5598152784719,1.89556559916036,"NA ","NA ",6.5,24.9108007765506,NA,74 +"2219","Atlantis","10",51.5868441792492,1.91576974637363,"NA ","NA ",6.5,23.4011255886579,NA,74 +"372","Atlantis","10",51.6122615419273,1.93348407431468,"NA ","NA ",6.5,212.425466046035,NA,74 +"672","Atlantis","10",51.5966323272854,1.91750249455364,"NA ","NA ",6.5,206.154300884336,NA,74 +"785","Atlantis","10",51.5711753807526,1.89739071432111,"NA ","NA ",6.5,205.25740051091,NA,74 +"877","Atlantis","10",51.5433568827942,1.87628911392041,"NA ","NA ",6.5,204.661709900562,NA,74 +"9415","Atlantis","10",51.5176675438312,1.85733557521998,"NA ","NA ",6.5,204.209581314852,NA,74 +"514","Atlantis","10",51.4776259807754,1.82827644594878,"NA ","NA ",6,206.166822798084,NA,75 +"115","Atlantis","10",51.4610835898273,1.81523147937276,"NA ","NA ",6,219.176969202052,NA,75 +"295","Atlantis","10",51.4484139591887,1.79866640370102,"NA ","NA ",6,9.81406530584997,NA,75 +"454","Atlantis","10",51.4658600475821,1.80351067891183,"NA ","NA ",6,17.7204984482591,NA,75 +"562","Atlantis","10",51.4825756293863,1.81208762027473,"NA ","NA ",6,26.742386402625,NA,75 +"7311","Atlantis","10",51.4988132754915,1.82523092346508,"NA ","NA ",6,174.839475088664,NA,75 +"904","Atlantis","10",51.4820854213066,1.82765679165919,"NA ","NA ",6,187.656521491711,NA,75 +"966","Atlantis","10",51.4637874750802,1.82370848236496,"NA ","NA ",6,189.47204391124,NA,75 +"617","Atlantis","10",51.4250912037577,1.81277099753234,"NA ","NA ",5.5,191.290926073399,NA,76 +"125","Atlantis","10",51.4018738041133,1.80534069604167,"NA ","NA ",5.5,192.572066143794,NA,76 +"206","Atlantis","10",51.376259587754,1.79618940669636,"NA ","NA ",5.5,194.471267575093,NA,76 +"296","Atlantis","10",51.3545769661917,1.78722891187607,"NA ","NA ",5.5,202.012207326746,NA,76 +"5418","Atlantis","10",51.3323073073975,1.77282005723227,"NA ","NA ",5.5,7.51701294652327,NA,76 +"763","Atlantis","10",51.3561783719977,1.77786413404218,"NA ","NA ",5.5,13.9671677146338,NA,76 +"864","Atlantis","10",51.3796281050854,1.78720872620325,"NA ","NA ",5.5,16.0522608694237,NA,76 +"9416","Atlantis","10",51.403788108627,1.79835246661946,"NA ","NA ",5.5,17.4426794037898,NA,76 +"515","Atlantis","10",51.4383637116139,1.81618819296893,"NA ","NA ",5,19.936127662946,NA,77 +"116","Atlantis","10",51.4529692373546,1.82468951460873,"NA ","NA ",5,25.7333771851634,NA,77 +"2220","Atlantis","10",51.4653152005451,1.83424147291835,"NA ","NA ",5,172.109392640864,NA,77 +"442","Atlantis","10",51.4533952629198,1.83689256744612,"NA ","NA ",5,188.029827611141,NA,77 +"563","Atlantis","10",51.4389069012487,1.83361367745341,"NA ","NA ",5,192.281779782827,NA,77 +"786","Atlantis","10",51.4264236975665,1.82925518580056,"NA ","NA ",5,8.1230086839056,NA,77 +"905","Atlantis","10",51.4401624523155,1.8324011028637,"NA ","NA ",5,9.49508408612189,NA,77 +"967","Atlantis","10",51.4554329900928,1.83649993390808,"NA ","NA ",5,9.87929792381539,NA,77 +"915","Atlantis","10",51.4917821683601,1.84635244823979,"NA ","NA ",6,210.136043557028,NA,78 +"3116","Atlantis","10",51.485456062957,1.84045511448566,"NA ","NA ",6,194.063974002827,NA,78 +"406","Atlantis","10",51.465011096713,1.83223401258828,"NA ","NA ",6,193.611345632429,NA,78 +"484","Atlantis","10",51.4434562423614,1.82386047552623,"NA ","NA ",6,193.53785827726,NA,78 +"564","Atlantis","10",51.4219301805742,1.81554891744367,"NA ","NA ",6,193.824478288262,NA,78 +"662","Atlantis","10",51.3997598075359,1.80680443955471,"NA ","NA ",6,227.136757010119,NA,78 +"8810","Atlantis","10",51.3977732924501,1.80337374743305,"NA ","NA ",6,10.364824478371,NA,78 +"9512","Atlantis","10",51.4185780093444,1.80947549211828,"NA ","NA ",6,10.8738735982964,NA,78 +"916","Atlantis","10",51.4753561530129,1.82996123715887,"NA ","NA ",7.5,59.579091657091,NA,79 +"304","Atlantis","10",51.48907325361,1.86749496555452,"NA ","NA ",7.5,145.324757944795,NA,79 +"443","Atlantis","10",51.46019005833,1.89955968407082,"NA ","NA ",7.5,147.095800463641,NA,79 +"553","Atlantis","10",51.4292194852903,1.93169561146274,"NA ","NA ",7.5,139.921572073982,NA,79 +"653","Atlantis","10",51.4023355386287,1.96794866412532,"NA ","NA ",7.5,124.789521661561,NA,79 +"753","Atlantis","10",51.3830658613426,2.01237054812977,"NA ","NA ",7.5,99.1764968495015,NA,79 +"855","Atlantis","10",51.3776075622857,2.06637941987569,"NA ","NA ",7.5,71.6037484296205,NA,79 +"9417","Atlantis","10",51.3896257319878,2.12435638523024,"NA ","NA ",7.5,55.0169800834095,NA,79 +"618","Atlantis","10",51.4275434918429,2.20624477902662,"NA ","NA ",6.5,55.2095109200365,NA,80 +"126","Atlantis","10",51.4460547824329,2.2490122687798,"NA ","NA ",6.5,59.2446005704021,NA,80 +"183","Atlantis","10",51.4607768383862,2.28873863714271,"NA ","NA ",6.5,63.5868933563512,NA,80 +"253","Atlantis","10",51.4738110592798,2.33089399920785,"NA ","NA ",6.5,68.4073350381941,NA,80 +"331","Atlantis","10",51.484189937102,2.37303572399053,"NA ","NA ",6.5,73.6791662834505,NA,80 +"444","Atlantis","10",51.492628637735,2.41937312848413,"NA ","NA ",6.5,73.836037213396,NA,80 +"682","Atlantis","10",51.5011297742472,2.46654199529531,"NA ","NA ",6.5,309.485667703331,NA,80 +"917","Atlantis","10",51.5200085724044,2.42970662496377,"NA ","NA ",6.5,303.53825838871,NA,80 +"516","Atlantis","10",51.544186824282,2.37082268139879,"NA ","NA ",3,301.342381733468,NA,81 +"918","Atlantis","10",51.5503775103773,2.35447276247752,"NA ","NA ",3,298.002267202737,NA,81 +"158","Atlantis","10",51.556758328762,2.33516819036854,"NA ","NA ",3,288.692880598604,NA,81 +"242","Atlantis","10",51.560653039871,2.31664525371804,"NA ","NA ",3,205.076770651172,NA,81 +"445","Atlantis","10",51.5517278386729,2.30992876009734,"NA ","NA ",3,150.815360421766,NA,81 +"554","Atlantis","10",51.5410574430063,2.31951068123154,"NA ","NA ",3,143.145685928834,NA,81 +"654","Atlantis","10",51.5306964048659,2.33199410656941,"NA ","NA ",3,137.427735679087,NA,81 +"772","Atlantis","10",51.5203542799481,2.34726149181014,"NA ","NA ",3,100.175325653814,NA,81 +"193","Atlantis","10",51.5120879589268,2.32538762843247,"NA ","NA ",3,231.516829158004,NA,82 +"297","Atlantis","10",51.4949203575992,2.29071083389133,"NA ","NA ",3,228.229320024648,NA,82 +"365","Atlantis","10",51.4795268148381,2.26304385117807,"NA ","NA ",3,226.459006970154,NA,82 +"433","Atlantis","10",51.4626373331253,2.23452383809344,"NA ","NA ",3,224.884212037635,NA,82 +"517","Atlantis","10",51.4429450685023,2.20306412332626,"NA ","NA ",3,223.277776080938,NA,82 +"582","Atlantis","10",51.4266346290268,2.17843635934384,"NA ","NA ",3,220.815797661444,NA,82 +"683","Atlantis","10",51.4071450165394,2.15145628756261,"NA ","NA ",3,181.186648849469,NA,82 +"919","Atlantis","10",51.3959142353963,2.15108343935834,"NA ","NA ",3,54.1833508015991,NA,82 +"117","Atlantis","10",51.4301052788398,2.22596229393461,"NA ","NA ",6,74.093165991656,NA,83 +"373","Atlantis","10",51.4370730463461,2.26522039718039,"NA ","NA ",6,207.620012814363,NA,83 +"555","Atlantis","10",51.4103735938885,2.24282494563002,"NA ","NA ",6,214.773422363393,NA,83 +"663","Atlantis","10",51.3833671270649,2.21278400718121,"NA ","NA ",6,216.440699980041,NA,83 +"742","Atlantis","10",51.3592755852687,2.18430114222184,"NA ","NA ",6,217.182272028617,NA,83 +"8110","Atlantis","10",51.335433394435,2.15535741917502,"NA ","NA ",6,217.612244770286,NA,83 +"8811","Atlantis","10",51.3092598483564,2.12310448344576,"NA ","NA ",6,217.855133898102,NA,83 +"9418","Atlantis","10",51.2851482401324,2.09314645228741,"NA ","NA ",6,217.97931650635,NA,83 +"518","Atlantis","10",51.2439625330956,2.04212785486628,"NA ","NA ",6.5,215.639170580411,NA,84 +"118","Atlantis","10",51.2265825756457,2.02223203902399,"NA ","NA ",6.5,194.996523510379,NA,84 +"3117","Atlantis","10",51.2109387549268,2.0155425232901,"NA ","NA ",6.5,51.3814338833755,NA,84 +"446","Atlantis","10",51.2239086744704,2.04147336579528,"NA ","NA ",6.5,45.3909827691995,NA,84 +"5419","Atlantis","10",51.2393449064211,2.06647231301613,"NA ","NA ",6.5,41.4327369248936,NA,84 +"643","Atlantis","10",51.2548931794727,2.08840264885308,"NA ","NA ",6.5,14.477351502674,NA,84 +"856","Atlantis","10",51.2691639235199,2.09429190252094,"NA ","NA ",6.5,240.471258826952,NA,84 +"9513","Atlantis","10",51.2582804909207,2.06360079801762,"NA ","NA ",6.5,235.008889635606,NA,84 +"519","Atlantis","10",51.3336607638984,2.22822132857179,"NA ","NA ",4,98.1390383394275,NA,86 +"106","Atlantis","10",51.3302798543159,2.26598771104413,"NA ","NA ",4,96.4748393217383,NA,86 +"159","Atlantis","10",51.3278888452763,2.29963573696313,"NA ","NA ",4,94.1424341521312,NA,86 +"2221","Atlantis","10",51.3260392185852,2.340348562195,"NA ","NA ",4,90.8449907660994,NA,86 +"298","Atlantis","10",51.3257215397934,2.37428505411787,"NA ","NA ",4,86.1572240439098,NA,86 +"38","Atlantis","10",51.3271808592892,2.40917694570008,"NA ","NA ",4,77.8277278963536,NA,86 +"5110","Atlantis","10",51.3320385767402,2.44526220424829,"NA ","NA ",4,62.5120021824563,NA,86 +"7111","Atlantis","10",51.3426499722131,2.47792542699389,"NA ","NA ",4,57.9052362817049,NA,86 +"305","Atlantis","10",51.361595069591,2.53755915684557,"NA ","NA ",3.5,61.8075467488202,NA,87 +"50","Atlantis","10",51.3707708761461,2.56498954654793,"NA ","NA ",3.5,63.621297493857,NA,87 +"632","Atlantis","10",51.3798522427578,2.59433929371153,"NA ","NA ",3.5,65.2866502841723,NA,87 +"728","Atlantis","10",51.3879572133019,2.622571331749,"NA ","NA ",3.5,66.3964523937344,NA,87 +"7919","Atlantis","10",51.3954374796724,2.65001956412045,"NA ","NA ",3.5,67.2009485871006,NA,87 +"857","Atlantis","10",51.4027466656194,2.67790569510775,"NA ","NA ",3.5,67.8646350980193,NA,87 +"9110","Atlantis","10",51.4109467212167,2.71024264515711,"NA ","NA ",3.5,68.4120632448985,NA,87 +"968","Atlantis","10",51.418503262643,2.74088192004486,"NA ","NA ",3.5,68.8208571896564,NA,87 +"619","Atlantis","10",51.432449331977,2.79803380486718,"NA ","NA ",5.5,65.6163614791254,NA,88 +"1510","Atlantis","10",51.4408945437488,2.82793794581874,"NA ","NA ",5.5,275.467143116053,NA,88 +"366","Atlantis","10",51.4420429520922,2.80866120177382,"NA ","NA ",5.5,256.499904862416,NA,88 +"455","Atlantis","10",51.437372269631,2.77747998803157,"NA ","NA ",5.5,254.811609219785,NA,88 +"533","Atlantis","10",51.431989388605,2.74570107359794,"NA ","NA ",5.5,253.504517164837,NA,88 +"6217","Atlantis","10",51.425602656494,2.71113927345336,"NA ","NA ",5.5,251.529409145227,NA,88 +"729","Atlantis","10",51.4194545710819,2.68164186928606,"NA ","NA ",5.5,125.243979391111,NA,88 +"939","Atlantis","10",51.4166926413691,2.68790926809247,"NA ","NA ",5.5,78.8317702301778,NA,88 +"730","Atlantis","10",51.4244989868684,2.75398261084055,"NA ","NA ",6.5,84.9525278308247,NA,89 +"1511","Atlantis","10",51.4262191365113,2.7852930629269,"NA ","NA ",6.5,101.504682037855,NA,89 +"332","Atlantis","10",51.4220579664231,2.81804247620535,"NA ","NA ",6.5,70.5437639807203,NA,89 +"7210","Atlantis","10",51.4268293418494,2.83971382836989,"NA ","NA ",6.5,43.1725370681207,NA,89 +"828","Atlantis","10",51.4428921235724,2.8638947093087,"NA ","NA ",6.5,44.6483813349979,NA,89 +"8812","Atlantis","10",51.4573349235024,2.88679452811889,"NA ","NA ",6.5,45.598523029913,NA,89 +"9310","Atlantis","10",51.4725386798362,2.91172263849778,"NA ","NA ",6.5,46.29772070689,NA,89 +"974","Atlantis","10",51.4869694398066,2.935975924477,"NA ","NA ",6.5,46.7945302062389,NA,89 +"520","Atlantis","10",51.5148143121442,2.98326955803542,"NA ","NA ",7.5,44.7398137876451,NA,90 +"107","Atlantis","10",51.5302263388006,3.0078237792129,"NA ","NA ",7.5,40.0302115514442,NA,90 +"194","Atlantis","10",51.5478724362349,3.03166308829333,"NA ","NA ",7.5,287.300659918046,NA,90 +"414","Atlantis","10",51.5513028648653,3.01394447568185,"NA ","NA ",7.5,239.590735873952,NA,90 +"534","Atlantis","10",51.5400965659468,2.98325669592147,"NA ","NA ",7.5,233.416465297953,NA,90 +"664","Atlantis","10",51.5268340433613,2.95454326371511,"NA ","NA ",7.5,102.047473733019,NA,90 +"898","Atlantis","10",51.5246940423993,2.97065102880562,"NA ","NA ",7.5,64.6740413620516,NA,90 +"9514","Atlantis","10",51.5330638679442,2.99909352982954,"NA ","NA ",7.5,62.6232937180385,NA,90 +"620","Atlantis","10",51.5559096238097,3.07036225795442,"NA ","NA ",6.5,64.8787721064498,NA,91 +"136","Atlantis","10",51.5694180977786,3.11674382852684,"NA ","NA ",6.5,67.5685532080797,NA,91 +"207","Atlantis","10",51.5795343624013,3.15620415550029,"NA ","NA ",6.5,71.3766182166245,NA,91 +"306","Atlantis","10",51.588764373865,3.20032764555346,"NA ","NA ",6.5,81.8084091482432,NA,91 +"491","Atlantis","10",51.5925982827571,3.24328567489987,"NA ","NA ",6.5,237.140519487145,NA,91 +"764","Atlantis","10",51.5784547360924,3.20806379218859,"NA ","NA ",6.5,253.908162845886,NA,91 +"865","Atlantis","10",51.5707450996127,3.16511009850576,"NA ","NA ",6.5,258.116703387497,NA,91 +"9419","Atlantis","10",51.5646680362403,3.11872244073802,"NA ","NA ",6.5,260.910237493155,NA,91 +"740","Atlantis","10",51.5554966965946,3.03860406013785,"NA ","NA ",6.5,240.76610353978,NA,92 +"184","Atlantis","10",51.5421068997243,3.00015377323488,"NA ","NA ",6.5,180.375239223681,NA,92 +"343","Atlantis","10",51.5185793525132,2.99990614738424,"NA ","NA ",6.5,131.479548496342,NA,92 +"4718","Atlantis","10",51.5019894845115,3.0300441984153,"NA ","NA ",6.5,109.005519588334,NA,92 +"605","Atlantis","10",51.4933711182832,3.0702009737337,"NA ","NA ",6.5,73.7387268246089,NA,92 +"754","Atlantis","10",51.5002544431412,3.10814297016884,"NA ","NA ",6.5,12.1399862942607,NA,92 +"878","Atlantis","10",51.5239521117027,3.11633616426516,"NA ","NA ",6.5,344.252417980996,NA,92 +"9515","Atlantis","10",51.5509848861004,3.10407698338701,"NA ","NA ",6.5,334.687849553127,NA,92 +"630","Atlantis","10",51.5939895559141,3.06945150477381,"NA ","NA ",6.5,332.46766096582,NA,93 +"127","Atlantis","10",51.6108549361168,3.05529350593346,"NA ","NA ",6.5,348.924673740257,NA,93 +"344","Atlantis","10",51.6145884996821,3.0541165516902,"NA ","NA ",6.5,150.383629091663,NA,93 +"456","Atlantis","10",51.5950800989789,3.07196692088396,"NA ","NA ",6.5,150.886391686993,NA,93 +"535","Atlantis","10",51.577605353172,3.08762564152632,"NA ","NA ",6.5,151.232515393312,NA,93 +"6218","Atlantis","10",51.5585311421453,3.10446838346729,"NA ","NA ",6.5,152.169805718644,NA,93 +"773","Atlantis","10",51.5385864054579,3.12139550949954,"NA ","NA ",6.5,327.893577128294,NA,93 +"9420","Atlantis","10",51.5541523936411,3.10568625685487,"NA ","NA ",6.5,329.848204009676,NA,93 +"743","Atlantis","10",51.5951751105366,3.06433568605026,"NA ","NA ",6,315.12186781501,NA,94 +"1512","Atlantis","10",51.6136428141789,3.03471544482433,"NA ","NA ",6,294.026841055058,NA,94 +"262","Atlantis","10",51.6240093223466,2.99723655879744,"NA ","NA ",6,256.998217380106,NA,94 +"407","Atlantis","10",51.6185842482576,2.95943787239354,"NA ","NA ",6,212.269533552204,NA,94 +"574","Atlantis","10",51.5965052696869,2.9369973183864,"NA ","NA ",6,161.507839728979,NA,94 +"7312","Atlantis","10",51.5742673681691,2.94896371341842,"NA ","NA ",6,110.434355142171,NA,94 +"866","Atlantis","10",51.5649858838742,2.98900887872298,"NA ","NA ",6,86.0664134428082,NA,94 +"9421","Atlantis","10",51.566712383924,3.02956523057162,"NA ","NA ",6,75.5067137024284,NA,94 +"633","Atlantis","10",51.5790262012329,3.10011364310991,"NA ","NA ",5.5,75.5166417767053,NA,95 +"128","Atlantis","10",51.5831939123318,3.12609748713707,"NA ","NA ",5.5,112.546916109413,NA,95 +"322","Atlantis","10",51.5792024311784,3.14156428420225,"NA ","NA ",5.5,237.259137909878,NA,95 +"447","Atlantis","10",51.5691633683076,3.1164524071745,"NA ","NA ",5.5,242.833312055284,NA,95 +"536","Atlantis","10",51.5608255655706,3.0903278813736,"NA ","NA ",5.5,246.55568612811,NA,95 +"634","Atlantis","10",51.5529937812825,3.06129625395679,"NA ","NA ",5.5,257.710140776835,NA,95 +"7920","Atlantis","10",51.5491493823791,3.03294291396182,"NA ","NA ",5.5,38.8960668988306,NA,95 +"9422","Atlantis","10",51.5625223154131,3.05029984395307,"NA ","NA ",5.5,52.557830109565,NA,95 +"829","Atlantis","10",51.5892132671781,3.10842195849905,"NA ","NA ",6,54.8351713252366,NA,96 +"171","Atlantis","10",51.6049796888667,3.14446775954468,"NA ","NA ",6,55.4272538078844,NA,96 +"2810","Atlantis","10",51.6189168019735,3.17705025358633,"NA ","NA ",6,49.4660291104623,NA,96 +"6110","Atlantis","10",51.6251919292265,3.18887221250839,"NA ","NA ",6,240.650613179754,NA,96 +"744","Atlantis","10",51.612878496298,3.15362318680097,"NA ","NA ",6,240.976689440012,NA,96 +"834","Atlantis","10",51.5994103662906,3.1145634683151,"NA ","NA ",6,241.311099222596,NA,96 +"899","Atlantis","10",51.5881735014454,3.08152756310357,"NA ","NA ",6,241.610657817043,NA,96 +"9516","Atlantis","10",51.5751234929466,3.04269436789728,"NA ","NA ",6,241.889153001334,NA,96 +"745","Atlantis","10",51.5479044885175,2.96031400409845,"NA ","NA ",7,242.36976627048,NA,97 +"147","Atlantis","10",51.531853011685,2.91105258485895,"NA ","NA ",7,242.559111084539,NA,97 +"216","Atlantis","10",51.5174340461568,2.86645472296668,"NA ","NA ",7,242.681216679232,NA,97 +"299","Atlantis","10",51.5031283813055,2.82198980118619,"NA ","NA ",7,242.650128471502,NA,97 +"408","Atlantis","10",51.4876014783188,2.77381190603928,"NA ","NA ",7,240.087041227194,NA,97 +"665","Atlantis","10",51.4728288164886,2.73260939143142,"NA ","NA ",7,68.706560021615,NA,97 +"858","Atlantis","10",51.4846804215368,2.78148199826629,"NA ","NA ",7,67.2584148983223,NA,97 +"9311","Atlantis","10",51.496147140128,2.82545459240665,"NA ","NA ",7,67.0375961534383,NA,97 +"635","Atlantis","10",51.5201286845364,2.91554991170008,"NA ","NA ",6,65.5883165714405,NA,98 +"129","Atlantis","10",51.5322587203119,2.95854051900998,"NA ","NA ",6,64.5064273905966,NA,98 +"185","Atlantis","10",51.5436794129429,2.99707314481736,"NA ","NA ",6,63.1795772259022,NA,98 +"263","Atlantis","10",51.5575977875956,3.04137841667951,"NA ","NA ",6,61.3840626601701,NA,98 +"351","Atlantis","10",51.5711441802655,3.081346481785,"NA ","NA ",6,58.5183026638659,NA,98 +"4719","Atlantis","10",51.5850564140478,3.11792521029581,"NA ","NA ",6,344.922560863774,NA,98 +"804","Atlantis","10",51.5914947203872,3.11513335763485,"NA ","NA ",6,244.409197890015,NA,98 +"923","Atlantis","10",51.5794370769306,3.07464269936245,"NA ","NA ",6,241.783650537896,NA,98 +"830","Atlantis","10",51.5539941968484,2.99792059419845,"NA ","NA ",6,244.344360456582,NA,99 +"168","Atlantis","10",51.5421925963933,2.95843630794267,"NA ","NA ",6,246.815681863219,NA,99 +"254","Atlantis","10",51.5318301561245,2.91956560753861,"NA ","NA ",6,249.862638573407,NA,99 +"367","Atlantis","10",51.5232758410431,2.88209780168919,"NA ","NA ",6,267.558360275579,NA,99 +"673","Atlantis","10",51.5226381569571,2.85815433432005,"NA ","NA ",6,67.1455271880295,NA,99 +"8111","Atlantis","10",51.5332501615368,2.89865598079138,"NA ","NA ",6,70.392456573366,NA,99 +"8910","Atlantis","10",51.5422385620095,2.93925735010021,"NA ","NA ",6,71.751105215708,NA,99 +"9517","Atlantis","10",51.5503451692141,2.978827593859,"NA ","NA ",6,72.6293677366713,NA,99 diff --git a/tacsatmini.csv b/tacsatmini.csv new file mode 100644 index 0000000..f3dd20a --- /dev/null +++ b/tacsatmini.csv @@ -0,0 +1,1001 @@ +"VE_COU","VE_REF","SI_LATI","SI_LONG","SI_DATE","SI_TIME","SI_SP","SI_HE","SI_DATIM" +"658237","Atlantis","10",51.4477244459512,3.58373072574753,"04/05/1800","13:32:00",0,25,1800-05-04 13:32:00 +"658238","Atlantis","10",51.4406743343966,3.58384667526838,"04/05/1800","15:28:00",0,25,1800-05-04 15:28:00 +"662888","Atlantis","10",51.4407431283058,3.59552922427561,"04/05/1800","17:22:00",0,25,1800-05-04 17:22:00 +"662889","Atlantis","10",51.4431543027703,3.58686242580321,"04/05/1800","19:18:00",0,25,1800-05-04 19:18:00 +"662890","Atlantis","10",51.4435144856712,3.58694753077719,"04/05/1800","21:12:00",0,25,1800-05-04 21:12:00 +"662891","Atlantis","10",51.4481111547514,3.59395834521763,"04/05/1800","23:08:00",0,25,1800-05-04 23:08:00 +"662892","Atlantis","10",51.4474296509796,3.59117479477078,"05/05/1800"," 1:04:00",0,111,1800-05-05 01:04:00 +"662893","Atlantis","10",51.6214494813131,3.19755943488237,"05/05/1800"," 2:58:00",12,292,1800-05-05 02:58:00 +"662894","Atlantis","10",51.6731934304331,3.17657612829283,"05/05/1800"," 4:54:00",2,43,1800-05-05 04:54:00 +"662895","Atlantis","10",51.6943234914839,3.22908421075065,"05/05/1800"," 6:48:00",2,294,1800-05-05 06:48:00 +"662896","Atlantis","10",51.6560659951191,3.16098873873707,"05/05/1800"," 8:44:00",6,144,1800-05-05 08:44:00 +"662897","Atlantis","10",51.6070417380915,3.08350550274644,"05/05/1800","10:40:00",7,248,1800-05-05 10:40:00 +"662898","Atlantis","10",51.576919331173,2.98779570994805,"05/05/1800","14:30:00",6,228,1800-05-05 14:30:00 +"662899","Atlantis","10",51.5465515189292,2.87639516843669,"05/05/1800","16:24:00",7,50,1800-05-05 16:24:00 +"662900","Atlantis","10",51.6061289456477,2.97909264942538,"05/05/1800","18:20:00",7,330,1800-05-05 18:20:00 +"662901","Atlantis","10",51.6016501706596,3.0204385781968,"05/05/1800","20:16:00",6,58,1800-05-05 20:16:00 +"662902","Atlantis","10",51.5731208998202,2.8929834927395,"05/05/1800","22:10:00",7,253,1800-05-05 22:10:00 +"662903","Atlantis","10",51.5931214836561,2.99407952163834,"06/05/1800","00: 6:00",7,245,1800-05-06 00:06:00 +"662904","Atlantis","10",51.5683078272836,2.93339168617967,"06/05/1800"," 2:00:00",7,330,1800-05-06 02:00:00 +"662905","Atlantis","10",51.5473403272065,2.87607746142615,"06/05/1800"," 3:56:00",7,55,1800-05-06 03:56:00 +"662906","Atlantis","10",51.5662122405847,2.93659450235404,"06/05/1800"," 5:52:00",7,83,1800-05-06 05:52:00 +"662907","Atlantis","10",51.6299493680494,3.036660480747,"06/05/1800"," 7:46:00",7,59,1800-05-06 07:46:00 +"662908","Atlantis","10",51.6359223276689,3.09941987676174,"06/05/1800"," 9:42:00",6,116,1800-05-06 09:42:00 +"662909","Atlantis","10",51.5344633132401,2.87679365528095,"06/05/1800","11:36:00",7,228,1800-05-06 11:36:00 +"662910","Atlantis","10",51.511504783609,2.31121867679339,"06/05/1800","13:32:00",14,271,1800-05-06 13:32:00 +"662911","Atlantis","10",51.4578128865957,1.82871440000553,"06/05/1800","15:28:00",6,172,1800-05-06 15:28:00 +"662912","Atlantis","10",51.5112028607992,1.8587873764392,"06/05/1800","17:22:00",7,25,1800-05-06 17:22:00 +"662913","Atlantis","10",51.6795650937799,1.95230285054259,"06/05/1800","19:18:00",7,20,1800-05-06 19:18:00 +"662914","Atlantis","10",51.695745846319,1.92481063032057,"06/05/1800","21:12:00",7,102,1800-05-06 21:12:00 +"662915","Atlantis","10",51.5305358059094,1.86397848288994,"06/05/1800","23:08:00",7,204,1800-05-06 23:08:00 +"662916","Atlantis","10",51.4609367726953,1.83341799937282,"07/05/1800"," 1:04:00",6,12,1800-05-07 01:04:00 +"662917","Atlantis","10",51.4061924462626,1.80183325022925,"07/05/1800"," 2:58:00",6,191,1800-05-07 02:58:00 +"662918","Atlantis","10",51.4650244918605,1.83690901399869,"07/05/1800"," 4:54:00",7,12,1800-05-07 04:54:00 +"662919","Atlantis","10",51.5882227651426,2.02424114330858,"07/05/1800"," 6:48:00",13,123,1800-05-07 06:48:00 +"662920","Atlantis","10",51.4148697609562,2.18524577266816,"07/05/1800"," 8:44:00",6,223,1800-05-07 08:44:00 +"662921","Atlantis","10",51.4457611120637,2.22457806055807,"07/05/1800","12:34:00",7,218,1800-05-07 12:34:00 +"662922","Atlantis","10",51.4589082118147,2.2554634172162,"07/05/1800","14:30:00",9,301,1800-05-07 14:30:00 +"662923","Atlantis","10",51.4187489186181,2.19479238274507,"07/05/1800","16:24:00",6,215,1800-05-07 16:24:00 +"662924","Atlantis","10",51.5047392145819,2.39375767114479,"07/05/1800","18:20:00",11,138,1800-05-07 18:20:00 +"662925","Atlantis","10",51.4934193430715,2.42827651779447,"08/05/1800"," 2:00:00",7,229,1800-05-08 02:00:00 +"662926","Atlantis","10",51.4012685797266,2.24624043547455,"08/05/1800"," 3:56:00",6,38,1800-05-08 03:56:00 +"662927","Atlantis","10",51.478767084531,2.40182553862966,"08/05/1800"," 5:52:00",5,230,1800-05-08 05:52:00 +"662928","Atlantis","10",51.4856400817088,2.40774057370703,"08/05/1800"," 7:46:00",7,43,1800-05-08 07:46:00 +"662929","Atlantis","10",51.5109846706307,2.51050425128173,"08/05/1800"," 9:42:00",6,203,1800-05-08 09:42:00 +"662930","Atlantis","10",51.3685016771392,2.30006208344549,"08/05/1800","11:36:00",6,191,1800-05-08 11:36:00 +"662931","Atlantis","10",51.4594513731385,2.34565509427059,"08/05/1800","13:32:00",7,236,1800-05-08 13:32:00 +"662932","Atlantis","10",51.4663128634985,2.3640575841507,"08/05/1800","17:22:00",7,56,1800-05-08 17:22:00 +"662933","Atlantis","10",51.3029056911548,2.4005533228144,"08/05/1800","19:18:00",7,37,1800-05-08 19:18:00 +"662934","Atlantis","10",51.3948131802687,2.64025816989597,"08/05/1800","21:12:00",7,74,1800-05-08 21:12:00 +"662935","Atlantis","10",51.4710751335397,2.92117160728946,"08/05/1800","23:08:00",11,136,1800-05-08 23:08:00 +"662936","Atlantis","10",51.4205238452363,3.47438484901283,"09/05/1800"," 1:04:00",11,82,1800-05-09 01:04:00 +"662937","Atlantis","10",51.4485953284753,3.59852260938287,"09/05/1800"," 2:58:00",0,15,1800-05-09 02:58:00 +"662938","Atlantis","10",51.4467330461042,3.58989608570375,"09/05/1800"," 4:54:00",0,15,1800-05-09 04:54:00 +"662939","Atlantis","10",51.4472292818362,3.5884271329958,"09/05/1800"," 6:48:00",0,15,1800-05-09 06:48:00 +"662940","Atlantis","10",51.4471316511449,3.58982776055485,"09/05/1800"," 8:44:00",0,15,1800-05-09 08:44:00 +"662941","Atlantis","10",51.4484958911571,3.58411605070718,"09/05/1800","12:34:00",0,15,1800-05-09 12:34:00 +"667604","Atlantis","10",51.4437598910858,3.583766422024,"11/05/1800","14:30:00",0,15,1800-05-11 14:30:00 +"667605","Atlantis","10",51.4416034219647,3.5851950481059,"11/05/1800","16:24:00",0,15,1800-05-11 16:24:00 +"667606","Atlantis","10",51.4472780093341,3.59583351112343,"11/05/1800","18:20:00",0,15,1800-05-11 18:20:00 +"667607","Atlantis","10",51.4445924110152,3.59097028739192,"11/05/1800","20:16:00",0,15,1800-05-11 20:16:00 +"667608","Atlantis","10",51.4429707641923,3.59499000045843,"11/05/1800","22:10:00",0,15,1800-05-11 22:10:00 +"667609","Atlantis","10",51.4448838578933,3.59818694050796,"12/05/1800","00: 6:00",0,15,1800-05-12 00:06:00 +"667610","Atlantis","10",51.4455564921806,3.55852451810613,"12/05/1800"," 2:00:00",14,311,1800-05-12 02:00:00 +"667611","Atlantis","10",51.6548137522223,3.14251614461281,"12/05/1800"," 5:52:00",7,218,1800-05-12 05:52:00 +"667612","Atlantis","10",51.5775022460828,2.90554459167179,"12/05/1800"," 9:42:00",7,60,1800-05-12 09:42:00 +"667613","Atlantis","10",51.6453192323837,2.95586224767007,"12/05/1800","11:36:00",7,336,1800-05-12 11:36:00 +"667614","Atlantis","10",51.911236426278,2.95896811183542,"12/05/1800","13:32:00",13,349,1800-05-12 13:32:00 +"667615","Atlantis","10",51.9453403444826,2.94993140023574,"12/05/1800","13:42:00",13,342,1800-05-12 13:42:00 +"667616","Atlantis","10",52.0593325713198,2.73492714592721,"12/05/1800","15:28:00",7,309,1800-05-12 15:28:00 +"667617","Atlantis","10",52.0947284897235,2.43497160246968,"12/05/1800","17:22:00",6,262,1800-05-12 17:22:00 +"667618","Atlantis","10",51.9565004344275,2.27307889834326,"12/05/1800","19:18:00",14,216,1800-05-12 19:18:00 +"667619","Atlantis","10",51.6590415419177,1.9370064862268,"12/05/1800","21:12:00",6,196,1800-05-12 21:12:00 +"667620","Atlantis","10",51.4842405747063,1.84669348298386,"12/05/1800","23:08:00",5,197,1800-05-12 23:08:00 +"667621","Atlantis","10",51.5091452060263,1.85640532855037,"13/05/1800"," 1:04:00",7,26,1800-05-13 01:04:00 +"667622","Atlantis","10",51.4929940155745,1.83951853084937,"13/05/1800"," 2:58:00",6,204,1800-05-13 02:58:00 +"667623","Atlantis","10",51.4471113794856,1.81924429896846,"13/05/1800"," 4:54:00",6,190,1800-05-13 04:54:00 +"667624","Atlantis","10",51.4249639321375,1.80902313761879,"13/05/1800"," 6:48:00",5,18,1800-05-13 06:48:00 +"667625","Atlantis","10",51.4693282687338,1.84038470214791,"13/05/1800"," 8:44:00",5,10,1800-05-13 08:44:00 +"667626","Atlantis","10",51.4418218042934,1.81663903429546,"13/05/1800","10:40:00",7,11,1800-05-13 10:40:00 +"667627","Atlantis","10",51.4088726234972,2.16847054663394,"13/05/1800","12:34:00",8,50,1800-05-13 12:34:00 +"667628","Atlantis","10",51.5362964495379,2.39018587551173,"13/05/1800","14:30:00",5,304,1800-05-13 14:30:00 +"667629","Atlantis","10",51.5190567561476,2.35887411012128,"13/05/1800","16:24:00",1,341,1800-05-13 16:24:00 +"667630","Atlantis","10",51.4108731082259,2.18432660974283,"13/05/1800","18:20:00",5,52,1800-05-13 18:20:00 +"667631","Atlantis","10",51.2596449704305,2.06133533269446,"13/05/1800","20:16:00",7,218,1800-05-13 20:16:00 +"667632","Atlantis","10",51.2466653574975,2.03709894624539,"13/05/1800","22:10:00",6,234,1800-05-13 22:10:00 +"667633","Atlantis","10",51.3371365424073,2.19485791365616,"14/05/1800"," 2:00:00",7,100,1800-05-14 02:00:00 +"667634","Atlantis","10",51.3545975816706,2.50843999481108,"14/05/1800"," 3:56:00",1,77,1800-05-14 03:56:00 +"667635","Atlantis","10",51.4250464622211,2.76797837713175,"14/05/1800"," 5:52:00",6,69,1800-05-14 05:52:00 +"667636","Atlantis","10",51.4208011545157,2.72131891132705,"14/05/1800"," 7:46:00",5,78,1800-05-14 07:46:00 +"667637","Atlantis","10",51.4992059450257,2.95690686668456,"14/05/1800"," 9:42:00",8,47,1800-05-14 09:42:00 +"667638","Atlantis","10",51.5439986988604,3.03306273283996,"14/05/1800","11:36:00",7,62,1800-05-14 11:36:00 +"667639","Atlantis","10",51.5604905423173,3.07679762857966,"14/05/1800","13:32:00",6,262,1800-05-14 13:32:00 +"667640","Atlantis","10",51.5730975413276,3.0872489113044,"14/05/1800","15:28:00",7,332,1800-05-14 15:28:00 +"667641","Atlantis","10",51.5740948810196,3.08704559132829,"14/05/1800","17:22:00",6,330,1800-05-14 17:22:00 +"667642","Atlantis","10",51.5730116140666,3.06881503879931,"14/05/1800","19:18:00",6,72,1800-05-14 19:18:00 +"667643","Atlantis","10",51.5742488432918,3.07494611991662,"14/05/1800","21:12:00",5,54,1800-05-14 21:12:00 +"667644","Atlantis","10",51.562859330554,3.00578095217701,"14/05/1800","23:08:00",7,242,1800-05-14 23:08:00 +"667645","Atlantis","10",51.5095643621851,2.87637615688983,"15/05/1800"," 1:04:00",7,67,1800-05-15 01:04:00 +"667646","Atlantis","10",51.566353250348,3.03543414315209,"15/05/1800"," 2:58:00",5,241,1800-05-15 02:58:00 +"667647","Atlantis","10",51.5579811066706,3.01812346521858,"15/05/1800"," 4:54:00",7,73,1800-05-15 04:54:00 +"667648","Atlantis","10",51.5684141412373,3.0783442192059,"15/05/1800"," 6:48:00",6,212,1800-05-15 06:48:00 +"667649","Atlantis","10",51.5484115438666,3.0216581239719,"15/05/1800"," 8:44:00",2,193,1800-05-15 08:44:00 +"667650","Atlantis","10",51.5452495538853,2.9637534877751,"15/05/1800","10:40:00",5,41,1800-05-15 10:40:00 +"667651","Atlantis","10",51.5808112133988,2.85450792735163,"15/05/1800","16:24:00",7,68,1800-05-15 16:24:00 +"667652","Atlantis","10",51.7043401395855,3.2504917239761,"15/05/1800","20:16:00",2,218,1800-05-15 20:16:00 +"667653","Atlantis","10",51.4437858781703,3.60456061182543,"15/05/1800","22:10:00",0,296,1800-05-15 22:10:00 +"667654","Atlantis","10",51.4469785857573,3.59578551058099,"16/05/1800","00: 6:00",0,356,1800-05-16 00:06:00 +"667655","Atlantis","10",51.4466890485212,3.59930284180119,"16/05/1800"," 2:00:00",0,25,1800-05-16 02:00:00 +"667656","Atlantis","10",51.4394142677062,3.5838732968187,"16/05/1800"," 3:56:00",0,189,1800-05-16 03:56:00 +"667657","Atlantis","10",51.4434437164781,3.58910195771232,"16/05/1800"," 5:52:00",0,189,1800-05-16 05:52:00 +"667658","Atlantis","10",51.4482281461498,3.58585640684329,"16/05/1800"," 7:46:00",0,189,1800-05-16 07:46:00 +"667659","Atlantis","10",51.4444931621128,3.59037094653491,"16/05/1800"," 9:42:00",0,354,1800-05-16 09:42:00 +"672312","Atlantis","10",51.4498283543345,3.59520274506323,"18/05/1800","17:22:00",0,354,1800-05-18 17:22:00 +"672313","Atlantis","10",51.4417422101041,3.58684735935088,"18/05/1800","19:18:00",0,354,1800-05-18 19:18:00 +"672314","Atlantis","10",51.4505019508409,3.60046504311077,"18/05/1800","21:12:00",0,354,1800-05-18 21:12:00 +"672315","Atlantis","10",51.4386785472436,3.58724097280763,"19/05/1800"," 1:04:00",0,110,1800-05-19 01:04:00 +"672316","Atlantis","10",51.6068690898926,3.2572747477619,"19/05/1800"," 2:58:00",12,291,1800-05-19 02:58:00 +"672317","Atlantis","10",51.6367579683163,3.12798582594376,"19/05/1800"," 4:54:00",6,133,1800-05-19 04:54:00 +"672318","Atlantis","10",51.6412097306293,3.12819190019648,"19/05/1800"," 6:48:00",4,312,1800-05-19 06:48:00 +"672319","Atlantis","10",51.6585236066496,3.09356418348011,"19/05/1800"," 8:44:00",6,315,1800-05-19 08:44:00 +"672320","Atlantis","10",51.6460921823192,3.15070162191056,"19/05/1800","10:40:00",6,117,1800-05-19 10:40:00 +"672321","Atlantis","10",51.6278278068313,3.1304722739188,"19/05/1800","12:34:00",6,91,1800-05-19 12:34:00 +"672322","Atlantis","10",51.5618644743208,2.91229306133464,"19/05/1800","14:30:00",6,236,1800-05-19 14:30:00 +"672323","Atlantis","10",51.5596606578669,2.94496564493887,"19/05/1800","16:24:00",7,57,1800-05-19 16:24:00 +"672324","Atlantis","10",51.5596441497481,2.93720259791147,"19/05/1800","16:24:00",7,57,1800-05-19 16:24:00 +"672325","Atlantis","10",51.5673459282885,2.92685810656566,"19/05/1800","18:20:00",7,36,1800-05-19 18:20:00 +"672326","Atlantis","10",51.5847009616452,2.98208248426113,"19/05/1800","20:16:00",6,53,1800-05-19 20:16:00 +"672327","Atlantis","10",51.5789075907171,2.9853329268042,"19/05/1800","20:16:00",6,53,1800-05-19 20:16:00 +"672328","Atlantis","10",51.6162410738356,3.07116217014753,"19/05/1800","22:10:00",6,349,1800-05-19 22:10:00 +"672329","Atlantis","10",51.5975333355735,3.02140312454756,"20/05/1800","00: 6:00",6,57,1800-05-20 00:06:00 +"672330","Atlantis","10",51.590457122311,3.00025474218279,"20/05/1800"," 2:00:00",7,250,1800-05-20 02:00:00 +"672331","Atlantis","10",51.5546195473284,2.94235223541502,"20/05/1800"," 3:56:00",6,47,1800-05-20 03:56:00 +"672332","Atlantis","10",51.5883381729759,3.00540564278979,"20/05/1800"," 5:52:00",6,61,1800-05-20 05:52:00 +"672333","Atlantis","10",51.6664156240979,3.15514422474522,"20/05/1800"," 7:46:00",3,25,1800-05-20 07:46:00 +"672334","Atlantis","10",51.6637506215745,3.1785451447973,"20/05/1800"," 9:42:00",5,305,1800-05-20 09:42:00 +"672335","Atlantis","10",51.6670854705386,3.12850809464138,"20/05/1800","11:36:00",7,199,1800-05-20 11:36:00 +"672336","Atlantis","10",51.5858145102956,3.07904149476718,"20/05/1800","13:32:00",6,53,1800-05-20 13:32:00 +"672337","Atlantis","10",51.5789679873553,3.08257830537669,"20/05/1800","13:32:00",6,53,1800-05-20 13:32:00 +"672338","Atlantis","10",51.5661317774118,3.08022928616777,"20/05/1800","15:28:00",8,254,1800-05-20 15:28:00 +"672339","Atlantis","10",51.5655843629269,3.09136122753192,"20/05/1800","15:28:00",8,254,1800-05-20 15:28:00 +"672340","Atlantis","10",51.6065265243943,3.0880650101006,"20/05/1800","17:22:00",7,99,1800-05-20 17:22:00 +"672341","Atlantis","10",51.6822944793711,3.23132003711164,"20/05/1800","21:12:00",7,299,1800-05-20 21:12:00 +"672342","Atlantis","10",51.3531013784846,2.52557839258388,"21/05/1800"," 4:54:00",2,70,1800-05-21 04:54:00 +"672343","Atlantis","10",51.3510980467331,2.5195066522155,"21/05/1800"," 6:48:00",2,128,1800-05-21 06:48:00 +"672344","Atlantis","10",51.4057357607242,1.98011851156689,"21/05/1800"," 8:44:00",11,276,1800-05-21 08:44:00 +"672345","Atlantis","10",51.6933935687239,1.91206538324058,"21/05/1800","12:34:00",5,56,1800-05-21 12:34:00 +"672346","Atlantis","10",51.5936210588138,1.90483856583666,"21/05/1800","14:30:00",6,122,1800-05-21 14:30:00 +"672347","Atlantis","10",51.4307533462876,1.8138914814638,"21/05/1800","16:24:00",10,138,1800-05-21 16:24:00 +"672348","Atlantis","10",51.4701913939635,1.8344310724251,"21/05/1800","18:20:00",6,9,1800-05-21 18:20:00 +"672349","Atlantis","10",51.6079437457426,1.91385515465308,"21/05/1800","20:16:00",7,3,1800-05-21 20:16:00 +"672350","Atlantis","10",51.5875464249258,2.24126899376046,"21/05/1800","22:10:00",12,122,1800-05-21 22:10:00 +"672351","Atlantis","10",51.4389535234203,2.22209049706813,"22/05/1800","00: 6:00",6,217,1800-05-22 00:06:00 +"672352","Atlantis","10",51.4003563606981,2.17254899724946,"22/05/1800"," 2:00:00",6,222,1800-05-22 02:00:00 +"672353","Atlantis","10",51.4411780570857,2.22556785787735,"22/05/1800"," 3:56:00",6,44,1800-05-22 03:56:00 +"672354","Atlantis","10",51.4443513077423,2.22165709033236,"22/05/1800"," 5:52:00",5,217,1800-05-22 05:52:00 +"672355","Atlantis","10",51.4379137101243,2.22858026063163,"22/05/1800"," 7:46:00",5,214,1800-05-22 07:46:00 +"672356","Atlantis","10",51.5139608935807,2.3422037399048,"22/05/1800"," 9:42:00",6,57,1800-05-22 09:42:00 +"672357","Atlantis","10",51.5011061795512,2.43439523145277,"22/05/1800","11:36:00",7,48,1800-05-22 11:36:00 +"672358","Atlantis","10",51.4499953094176,2.33965058990195,"22/05/1800","13:32:00",6,24,1800-05-22 13:32:00 +"672359","Atlantis","10",51.3898783241338,2.24577099526953,"22/05/1800","15:28:00",3,13,1800-05-22 15:28:00 +"672360","Atlantis","10",51.5051320922324,2.44321047574468,"22/05/1800","19:18:00",7,49,1800-05-22 19:18:00 +"672361","Atlantis","10",51.5075685075247,2.44582821303233,"22/05/1800","19:18:00",7,49,1800-05-22 19:18:00 +"672362","Atlantis","10",51.545395918095,2.63815955818817,"22/05/1800","21:12:00",4,198,1800-05-22 21:12:00 +"672363","Atlantis","10",51.5958505059853,2.74833332258091,"22/05/1800","23:08:00",8,94,1800-05-22 23:08:00 +"672364","Atlantis","10",51.4163502259934,3.22519611601811,"23/05/1800"," 1:04:00",11,79,1800-05-23 01:04:00 +"672365","Atlantis","10",51.4442419000026,3.58922122503072,"23/05/1800"," 2:58:00",4,322,1800-05-23 02:58:00 +"672366","Atlantis","10",51.4487870216602,3.59106035409402,"23/05/1800"," 4:54:00",0,92,1800-05-23 04:54:00 +"672367","Atlantis","10",51.4488727705739,3.59884215049352,"23/05/1800"," 6:48:00",0,92,1800-05-23 06:48:00 +"672368","Atlantis","10",51.4435630390584,3.58929342525918,"23/05/1800"," 8:44:00",0,92,1800-05-23 08:44:00 +"676983","Atlantis","10",51.4425128327706,3.59011167568993,"23/05/1800","10:40:00",0,92,1800-05-23 10:40:00 +"676984","Atlantis","10",51.443881428598,3.58982501373068,"23/05/1800","12:34:00",0,92,1800-05-23 12:34:00 +"677008","Atlantis","10",51.4413676513359,3.5852599167116,"25/05/1800","18:20:00",0,92,1800-05-25 18:20:00 +"677009","Atlantis","10",51.4454871094599,3.58342768724915,"25/05/1800","20:16:00",0,92,1800-05-25 20:16:00 +"677010","Atlantis","10",51.4472130877082,3.58618369957339,"25/05/1800","22:10:00",0,92,1800-05-25 22:10:00 +"677011","Atlantis","10",51.4405778946774,3.59741392126679,"26/05/1800","00: 6:00",0,92,1800-05-26 00:06:00 +"677012","Atlantis","10",51.5965147950058,3.31778014119342,"26/05/1800"," 2:00:00",12,292,1800-05-26 02:00:00 +"677013","Atlantis","10",51.679514797824,3.22658743065875,"26/05/1800"," 3:56:00",7,215,1800-05-26 03:56:00 +"677014","Atlantis","10",51.6534445190164,3.15240749916621,"26/05/1800"," 5:52:00",6,203,1800-05-26 05:52:00 +"677015","Atlantis","10",51.6226765899486,3.11763341970555,"26/05/1800"," 7:46:00",7,179,1800-05-26 07:46:00 +"677016","Atlantis","10",51.5529266411574,2.91942515312974,"26/05/1800"," 9:42:00",6,27,1800-05-26 09:42:00 +"677017","Atlantis","10",51.5761613158053,2.98066639199201,"26/05/1800","11:36:00",7,51,1800-05-26 11:36:00 +"677018","Atlantis","10",51.586419825085,3.00868589759059,"26/05/1800","13:32:00",7,49,1800-05-26 13:32:00 +"677019","Atlantis","10",51.5888217586107,3.03118455032259,"26/05/1800","15:28:00",6,54,1800-05-26 15:28:00 +"677020","Atlantis","10",51.5710880040219,2.93852564639505,"26/05/1800","17:22:00",6,64,1800-05-26 17:22:00 +"677021","Atlantis","10",51.558819605655,2.96253412400465,"26/05/1800","19:18:00",6,67,1800-05-26 19:18:00 +"677022","Atlantis","10",51.5459717204808,2.89160391475074,"26/05/1800","21:12:00",1,38,1800-05-26 21:12:00 +"677023","Atlantis","10",51.5362278128476,2.89195539133157,"26/05/1800","21:12:00",NA,NA,1800-05-26 21:12:00 +"677024","Atlantis","10",51.5609061656599,2.87349012409058,"26/05/1800","23:08:00",NA,NA,1800-05-26 23:08:00 +"677025","Atlantis","10",51.5614489930412,2.87179454894923,"26/05/1800","23:08:00",6,42,1800-05-26 23:08:00 +"677026","Atlantis","10",51.7253597725076,3.01744475515373,"27/05/1800"," 1:04:00",7,69,1800-05-27 01:04:00 +"677027","Atlantis","10",51.7456746744616,3.07719044709206,"27/05/1800"," 2:58:00",6,234,1800-05-27 02:58:00 +"677028","Atlantis","10",51.7155772482995,3.12027825713158,"27/05/1800"," 4:54:00",6,235,1800-05-27 04:54:00 +"677029","Atlantis","10",51.5984315578206,2.96170603665989,"27/05/1800"," 6:48:00",6,59,1800-05-27 06:48:00 +"677030","Atlantis","10",51.6127899034694,3.01407608689554,"27/05/1800"," 8:44:00",8,246,1800-05-27 08:44:00 +"677031","Atlantis","10",51.4931282767719,2.81815157718118,"27/05/1800","10:40:00",6,257,1800-05-27 10:40:00 +"677032","Atlantis","10",51.4981504839892,2.33773900338728,"27/05/1800","12:34:00",12,276,1800-05-27 12:34:00 +"677033","Atlantis","10",51.5770034613227,1.90571926696598,"27/05/1800","14:30:00",7,26,1800-05-27 14:30:00 +"677034","Atlantis","10",51.67751587275,1.94941334740631,"27/05/1800","16:24:00",7,21,1800-05-27 16:24:00 +"677035","Atlantis","10",51.6778512750175,1.95051395815238,"27/05/1800","16:24:00",7,21,1800-05-27 16:24:00 +"677036","Atlantis","10",51.6265724248295,1.92285628780909,"27/05/1800","18:20:00",7,181,1800-05-27 18:20:00 +"677037","Atlantis","10",51.4901789905271,1.84238064859156,"27/05/1800","20:16:00",5,25,1800-05-27 20:16:00 +"677038","Atlantis","10",51.4941483656736,1.8341560415607,"27/05/1800","20:16:00",NA,NA,1800-05-27 20:16:00 +"677039","Atlantis","10",51.4925464702961,1.84628458611108,"28/05/1800"," 2:00:00",7,206,1800-05-28 02:00:00 +"677040","Atlantis","10",51.4557961362796,1.81918313868251,"28/05/1800"," 3:56:00",6,187,1800-05-28 03:56:00 +"677041","Atlantis","10",51.5044706184217,1.85029320965148,"28/05/1800"," 5:52:00",6,27,1800-05-28 05:52:00 +"677042","Atlantis","10",51.5158075350937,1.84238087548316,"28/05/1800"," 7:46:00",7,199,1800-05-28 07:46:00 +"677043","Atlantis","10",51.427995924768,1.81726581050921,"28/05/1800"," 9:42:00",6,16,1800-05-28 09:42:00 +"677044","Atlantis","10",51.4963966847067,1.85540833191294,"28/05/1800","11:36:00",7,22,1800-05-28 11:36:00 +"677045","Atlantis","10",51.6439060433051,1.92130181420967,"28/05/1800","13:32:00",7,11,1800-05-28 13:32:00 +"677046","Atlantis","10",51.6822505572815,1.93689612194802,"28/05/1800","15:28:00",6,135,1800-05-28 15:28:00 +"677047","Atlantis","10",51.6451249079532,2.0047009931244,"28/05/1800","17:22:00",13,106,1800-05-28 17:22:00 +"677048","Atlantis","10",51.5224785884791,2.19522223338578,"28/05/1800","19:18:00",6,37,1800-05-28 19:18:00 +"677049","Atlantis","10",51.5299558686926,2.20393963364977,"28/05/1800","21:14:00",7,218,1800-05-28 21:14:00 +"677050","Atlantis","10",51.5385962066553,2.21860907216091,"28/05/1800","23:08:00",6,38,1800-05-28 23:08:00 +"677051","Atlantis","10",51.4697996221241,2.41705975234602,"29/05/1800"," 1:04:00",14,90,1800-05-29 01:04:00 +"677052","Atlantis","10",51.4062104156092,3.10094109585975,"29/05/1800"," 2:58:00",13,102,1800-05-29 02:58:00 +"677053","Atlantis","10",51.4411262052413,3.59029239661433,"29/05/1800"," 4:54:00",1,268,1800-05-29 04:54:00 +"677054","Atlantis","10",51.4428056301554,3.59224467536621,"29/05/1800"," 6:48:00",0,49,1800-05-29 06:48:00 +"677055","Atlantis","10",51.4454917158089,3.59597365520615,"29/05/1800"," 8:44:00",0,49,1800-05-29 08:44:00 +"681675","Atlantis","10",51.4516734176693,3.58644658665825,"29/05/1800","10:40:00",0,49,1800-05-29 10:40:00 +"681676","Atlantis","10",51.4495323975226,3.58543850317318,"29/05/1800","12:34:00",0,49,1800-05-29 12:34:00 +"681677","Atlantis","10",51.4467095976421,3.60253330370132,"29/05/1800","14:30:00",0,49,1800-05-29 14:30:00 +"681678","Atlantis","10",51.4497009727526,3.59032916679699,"29/05/1800","16:24:00",0,49,1800-05-29 16:24:00 +"1547480","Atlantis","10",51.4471215216001,3.5874665289782,"05/09/1801","15:46:00",0,352,1801-09-05 15:46:00 +"1547481","Atlantis","10",51.4472549664276,3.59630610466376,"05/09/1801","17:42:00",0,352,1801-09-05 17:42:00 +"1547482","Atlantis","10",51.4489429760189,3.58790835902467,"05/09/1801","19:38:00",0,352,1801-09-05 19:38:00 +"1547483","Atlantis","10",51.4493196887663,3.58770060952194,"05/09/1801","21:32:00",0,352,1801-09-05 21:32:00 +"1547484","Atlantis","10",51.4445854990501,3.58647985284589,"05/09/1801","23:28:00",0,352,1801-09-05 23:28:00 +"1547485","Atlantis","10",51.4398813429773,3.59849998850748,"06/09/1801"," 1:22:00",0,114,1801-09-06 01:22:00 +"1547486","Atlantis","10",51.6479286842844,3.25952713057864,"06/09/1801"," 3:18:00",12,305,1801-09-06 03:18:00 +"1547487","Atlantis","10",51.6281376386625,3.0386842816649,"06/09/1801"," 7:08:00",7,66,1801-09-06 07:08:00 +"1547488","Atlantis","10",51.6509485563682,3.08952733156458,"06/09/1801"," 9:04:00",6,300,1801-09-06 09:04:00 +"1547489","Atlantis","10",51.6481914477092,3.14860756535083,"06/09/1801","10:58:00",5,129,1801-09-06 10:58:00 +"1547490","Atlantis","10",51.6529306280897,3.14159676210675,"06/09/1801","14:48:00",6,132,1801-09-06 14:48:00 +"1547491","Atlantis","10",51.670379133218,3.19591958882101,"06/09/1801","16:44:00",6,125,1801-09-06 16:44:00 +"1547492","Atlantis","10",51.6716838364308,3.19370284848195,"06/09/1801","16:44:00",0,58,1801-09-06 16:44:00 +"1547493","Atlantis","10",51.6817619224284,3.2123439415684,"06/09/1801","18:40:00",6,325,1801-09-06 18:40:00 +"1547494","Atlantis","10",51.6763011609525,3.1989973416226,"06/09/1801","18:40:00",6,325,1801-09-06 18:40:00 +"1547495","Atlantis","10",51.6873501892756,3.23244109593984,"06/09/1801","20:34:00",7,147,1801-09-06 20:34:00 +"1547496","Atlantis","10",51.6871790735023,3.23096863422263,"06/09/1801","20:34:00",7,147,1801-09-06 20:34:00 +"1547497","Atlantis","10",51.6851985401819,3.22443534885813,"06/09/1801","22:30:00",6,167,1801-09-06 22:30:00 +"1547498","Atlantis","10",51.6974421805986,3.25139044718072,"07/09/1801","00:26:00",7,320,1801-09-07 00:26:00 +"1547499","Atlantis","10",51.697171198735,3.23995037373155,"07/09/1801"," 2:20:00",7,81,1801-09-07 02:20:00 +"1547500","Atlantis","10",51.6978131556478,3.24973523151688,"07/09/1801"," 4:16:00",1,182,1801-09-07 04:16:00 +"1547501","Atlantis","10",51.7014432861628,3.24942585768644,"07/09/1801"," 6:10:00",4,305,1801-09-07 06:10:00 +"1547502","Atlantis","10",51.6987577115968,3.23692428721488,"07/09/1801"," 8:06:00",6,237,1801-09-07 08:06:00 +"1547503","Atlantis","10",51.7838488895763,3.29683520968538,"07/09/1801","10:02:00",6,206,1801-09-07 10:02:00 +"1547504","Atlantis","10",51.8068421097542,3.32008304503187,"07/09/1801","11:56:00",6,204,1801-09-07 11:56:00 +"1547505","Atlantis","10",51.7046374504543,3.24381653071474,"07/09/1801","13:52:00",7,207,1801-09-07 13:52:00 +"1547506","Atlantis","10",51.5438197602029,3.04950336607546,"07/09/1801","15:46:00",7,234,1801-09-07 15:46:00 +"1547507","Atlantis","10",51.4303528347453,2.74918987647444,"07/09/1801","17:42:00",6,294,1801-09-07 17:42:00 +"1547508","Atlantis","10",51.3394456307925,2.51056399023999,"07/09/1801","19:36:00",6,212,1801-09-07 19:36:00 +"1547509","Atlantis","10",51.3166115203057,2.41250956253149,"07/09/1801","21:32:00",9,350,1801-09-07 21:32:00 +"1547510","Atlantis","10",51.2819146101475,2.37883586782962,"07/09/1801","23:28:00",2,223,1801-09-07 23:28:00 +"1547511","Atlantis","10",51.2798472953723,2.37747471261769,"07/09/1801","23:28:00",2,223,1801-09-07 23:28:00 +"1547512","Atlantis","10",51.2353007429149,2.00376235204376,"08/09/1801"," 1:22:00",7,256,1801-09-08 01:22:00 +"1547513","Atlantis","10",51.2427454718272,2.01149504890945,"08/09/1801"," 3:18:00",5,322,1801-09-08 03:18:00 +"1547514","Atlantis","10",51.2019989012559,1.89096347641386,"08/09/1801"," 5:14:00",7,240,1801-09-08 05:14:00 +"1547515","Atlantis","10",51.2277650822424,1.95771768193971,"08/09/1801"," 7:08:00",7,39,1801-09-08 07:08:00 +"1547516","Atlantis","10",51.2359607488536,1.98193690547533,"08/09/1801"," 9:04:00",7,242,1801-09-08 09:04:00 +"1547517","Atlantis","10",51.2512144695586,2.03653786873538,"08/09/1801","10:58:00",7,38,1801-09-08 10:58:00 +"1547518","Atlantis","10",51.2490607190146,2.01211349516548,"08/09/1801","12:54:00",5,53,1801-09-08 12:54:00 +"1547519","Atlantis","10",51.2067375236326,1.901906176758,"08/09/1801","16:44:00",7,148,1801-09-08 16:44:00 +"1547520","Atlantis","10",51.2229271445018,1.92765690828487,"08/09/1801","18:40:00",6,60,1801-09-08 18:40:00 +"1547521","Atlantis","10",51.2217695461111,1.93127365481388,"08/09/1801","18:40:00",6,60,1801-09-08 18:40:00 +"1547522","Atlantis","10",51.2184366785004,1.941739888818,"08/09/1801","18:40:00",6,60,1801-09-08 18:40:00 +"1547523","Atlantis","10",51.3302821980892,2.16700047973357,"08/09/1801","20:34:00",6,37,1801-09-08 20:34:00 +"1547524","Atlantis","10",51.3502505001551,2.19771695313882,"08/09/1801","22:30:00",7,189,1801-09-08 22:30:00 +"1550103","Atlantis","10",51.4034635625151,2.25437117671408,"09/09/1801","00:26:00",6,80,1801-09-09 00:26:00 +"1550104","Atlantis","10",51.322973999144,2.150665619337,"09/09/1801"," 2:20:00",6,237,1801-09-09 02:20:00 +"1550105","Atlantis","10",51.222391182302,1.93384957790095,"09/09/1801"," 4:16:00",6,112,1801-09-09 04:16:00 +"1550106","Atlantis","10",51.2196073813797,1.92921392862871,"09/09/1801"," 4:16:00",6,112,1801-09-09 04:16:00 +"1550107","Atlantis","10",51.2129375260947,1.90642654551938,"09/09/1801"," 6:10:00",3,235,1801-09-09 06:10:00 +"1550108","Atlantis","10",51.2305429565869,1.92083688542153,"09/09/1801"," 8:06:00",7,88,1801-09-09 08:06:00 +"1550109","Atlantis","10",51.2666672397051,2.02879456453118,"09/09/1801","10:02:00",6,159,1801-09-09 10:02:00 +"1550110","Atlantis","10",51.3289543471597,2.1834359162394,"09/09/1801","11:56:00",7,47,1801-09-09 11:56:00 +"1550111","Atlantis","10",51.4675729378844,2.39237256994657,"09/09/1801","13:52:00",7,18,1801-09-09 13:52:00 +"1550112","Atlantis","10",51.461545520769,2.39517988363002,"09/09/1801","13:52:00",7,18,1801-09-09 13:52:00 +"1550113","Atlantis","10",51.5817346513038,2.70298299446236,"09/09/1801","15:46:00",12,71,1801-09-09 15:46:00 +"1550114","Atlantis","10",51.70327139585,3.12450265863724,"09/09/1801","17:42:00",6,56,1801-09-09 17:42:00 +"1550115","Atlantis","10",51.7465460068672,3.30236414455436,"09/09/1801","19:36:00",6,29,1801-09-09 19:36:00 +"1550116","Atlantis","10",51.7407954454753,3.22458097548038,"09/09/1801","21:32:00",6,245,1801-09-09 21:32:00 +"1550117","Atlantis","10",51.6340549074784,3.11983212189563,"09/09/1801","23:28:00",7,250,1801-09-09 23:28:00 +"1550118","Atlantis","10",51.4817744424981,3.49791612724587,"10/09/1801"," 1:22:00",12,139,1801-09-10 01:22:00 +"1550119","Atlantis","10",51.4463830043748,3.5825653932374,"10/09/1801"," 3:18:00",0,4,1801-09-10 03:18:00 +"1550120","Atlantis","10",51.4447868594038,3.59198094516899,"10/09/1801"," 5:12:00",0,4,1801-09-10 05:12:00 +"1550121","Atlantis","10",51.4470107603027,3.58766982076131,"10/09/1801"," 7:08:00",0,4,1801-09-10 07:08:00 +"1550122","Atlantis","10",51.4420946024871,3.59441434751172,"10/09/1801"," 9:04:00",0,4,1801-09-10 09:04:00 +"1550123","Atlantis","10",51.4428739119833,3.59144667406008,"10/09/1801","10:58:00",0,4,1801-09-10 10:58:00 +"1550124","Atlantis","10",51.4442425165256,3.58801649141032,"10/09/1801","12:54:00",0,4,1801-09-10 12:54:00 +"1550148","Atlantis","10",51.440633860731,3.59315317070484,"12/09/1801","16:44:00",0,4,1801-09-12 16:44:00 +"1550149","Atlantis","10",51.4435586108244,3.59003407027945,"12/09/1801","18:40:00",0,4,1801-09-12 18:40:00 +"1550150","Atlantis","10",51.443760890679,3.58849386924319,"12/09/1801","20:34:00",0,4,1801-09-12 20:34:00 +"1550151","Atlantis","10",51.4478253994952,3.596914519988,"12/09/1801","22:30:00",0,4,1801-09-12 22:30:00 +"1550152","Atlantis","10",51.4415832231264,3.58387214612588,"13/09/1801","00:26:00",0,4,1801-09-13 00:26:00 +"1550153","Atlantis","10",51.5560800412539,3.40860676246788,"13/09/1801"," 2:20:00",13,342,1801-09-13 02:20:00 +"1550154","Atlantis","10",51.6506954165152,3.10684370871633,"13/09/1801"," 4:16:00",6,142,1801-09-13 04:16:00 +"1550155","Atlantis","10",51.6603912329171,3.13684440137353,"13/09/1801"," 6:10:00",7,326,1801-09-13 06:10:00 +"1550156","Atlantis","10",51.7078825420258,3.03145686939359,"13/09/1801"," 8:06:00",6,217,1801-09-13 08:06:00 +"1550157","Atlantis","10",51.5953640259528,2.96931218023133,"13/09/1801","10:02:00",6,332,1801-09-13 10:02:00 +"1550158","Atlantis","10",51.5944646433513,2.98598895595595,"13/09/1801","10:02:00",3,2,1801-09-13 10:02:00 +"1550159","Atlantis","10",51.5903429940022,2.99568230347335,"13/09/1801","11:56:00",7,350,1801-09-13 11:56:00 +"1550160","Atlantis","10",51.593638931362,2.98919582143519,"13/09/1801","11:56:00",7,350,1801-09-13 11:56:00 +"1550161","Atlantis","10",51.6523126868028,3.10441637480073,"13/09/1801","13:52:00",7,53,1801-09-13 13:52:00 +"1550162","Atlantis","10",51.8188790104301,3.04491055795364,"13/09/1801","15:46:00",7,340,1801-09-13 15:46:00 +"1550163","Atlantis","10",52.0010131651941,2.9353805143265,"13/09/1801","17:42:00",4,184,1801-09-13 17:42:00 +"1550164","Atlantis","10",52.0411264571841,2.94222063376568,"13/09/1801","19:38:00",5,63,1801-09-13 19:38:00 +"1550165","Atlantis","10",51.9841536738188,2.89393158707861,"13/09/1801","21:32:00",6,62,1801-09-13 21:32:00 +"1550166","Atlantis","10",51.9448920345618,3.00449599343538,"13/09/1801","23:28:00",7,126,1801-09-13 23:28:00 +"1550167","Atlantis","10",51.9292196648321,3.05030322241504,"14/09/1801"," 1:22:00",5,261,1801-09-14 01:22:00 +"1550168","Atlantis","10",51.9546804678007,3.00834763032384,"14/09/1801"," 3:18:00",4,166,1801-09-14 03:18:00 +"1550169","Atlantis","10",51.9349663437787,3.06636027573794,"14/09/1801"," 5:12:00",5,194,1801-09-14 05:12:00 +"1550170","Atlantis","10",51.9503624857804,2.99984485744871,"14/09/1801"," 7:08:00",1,50,1801-09-14 07:08:00 +"1550171","Atlantis","10",51.8076397209452,3.07649357305467,"14/09/1801"," 9:04:00",6,205,1801-09-14 09:04:00 +"1550172","Atlantis","10",51.6352427168596,3.15532667065319,"14/09/1801","10:58:00",7,223,1801-09-14 10:58:00 +"1550173","Atlantis","10",51.5780256870347,3.09939626664575,"14/09/1801","12:54:00",3,158,1801-09-14 12:54:00 +"1550174","Atlantis","10",51.6252489654711,3.12505094688013,"14/09/1801","14:50:00",2,33,1801-09-14 14:50:00 +"1552741","Atlantis","10",51.758331616886,3.31563718857337,"14/09/1801","16:44:00",8,29,1801-09-14 16:44:00 +"1552742","Atlantis","10",51.9289247930767,3.3026310300082,"14/09/1801","18:40:00",8,239,1801-09-14 18:40:00 +"1552743","Atlantis","10",51.885049523632,3.29192748449184,"14/09/1801","20:34:00",6,269,1801-09-14 20:34:00 +"1552744","Atlantis","10",51.8625549719837,3.283867040121,"14/09/1801","22:30:00",1,189,1801-09-14 22:30:00 +"1552745","Atlantis","10",51.7350381040433,3.28467455265299,"15/09/1801","00:24:00",7,181,1801-09-15 00:24:00 +"1552746","Atlantis","10",51.6980518250223,3.25246431659069,"15/09/1801"," 2:20:00",6,151,1801-09-15 02:20:00 +"1552747","Atlantis","10",51.6917256179377,3.2417630917579,"15/09/1801"," 4:16:00",6,327,1801-09-15 04:16:00 +"1552748","Atlantis","10",51.6851408106345,3.25651520087663,"15/09/1801"," 6:10:00",2,64,1801-09-15 06:10:00 +"1552749","Atlantis","10",51.6880665467288,3.20573230723571,"15/09/1801"," 8:06:00",7,112,1801-09-15 08:06:00 +"1552750","Atlantis","10",51.5565556498272,3.06820806097053,"15/09/1801","10:02:00",7,240,1801-09-15 10:02:00 +"1552751","Atlantis","10",51.4365906517385,2.78492021421157,"15/09/1801","11:56:00",7,231,1801-09-15 11:56:00 +"1552752","Atlantis","10",51.3438614295041,2.31500267245341,"15/09/1801","15:46:00",6,273,1801-09-15 15:46:00 +"1552753","Atlantis","10",51.3456505737547,2.29505123374984,"15/09/1801","15:52:00",6,331,1801-09-15 15:52:00 +"1552754","Atlantis","10",51.3535146802477,2.29615095210448,"15/09/1801","15:54:00",6,335,1801-09-15 15:54:00 +"1552755","Atlantis","10",51.5142760261507,2.45296865792852,"15/09/1801","17:42:00",6,57,1801-09-15 17:42:00 +"1552756","Atlantis","10",51.5572330715698,2.52904815525655,"15/09/1801","19:38:00",6,200,1801-09-15 19:38:00 +"1552757","Atlantis","10",51.488495921168,2.40751826915518,"15/09/1801","21:32:00",7,225,1801-09-15 21:32:00 +"1552758","Atlantis","10",51.3399569089254,2.19267187903449,"15/09/1801","23:28:00",7,190,1801-09-15 23:28:00 +"1552759","Atlantis","10",51.247439417942,2.00999379438162,"16/09/1801"," 4:00:00",NA,NA,1801-09-16 04:00:00 +"1552760","Atlantis","10",51.2341361845899,2.01905191429146,"16/09/1801"," 4:10:00",5,207,1801-09-16 04:10:00 +"1552761","Atlantis","10",51.2699721660782,2.05125782146491,"16/09/1801"," 5:14:00",5.4,196,1801-09-16 05:14:00 +"1552762","Atlantis","10",51.2652179177795,2.04115732903034,"16/09/1801"," 5:14:00",7,30,1801-09-16 05:14:00 +"1552763","Atlantis","10",51.4107242681323,2.1807154715918,"16/09/1801"," 7:08:00",7,39,1801-09-16 07:08:00 +"1552764","Atlantis","10",51.4056691547669,2.19061063135974,"16/09/1801","10:58:00",6,221,1801-09-16 10:58:00 +"1552765","Atlantis","10",51.3010033387663,2.40550455520488,"16/09/1801","12:54:00",7,197,1801-09-16 12:54:00 +"1552766","Atlantis","10",51.3449685856211,2.51781653188448,"16/09/1801","14:50:00",5,216,1801-09-16 14:50:00 +"1552805","Atlantis","10",51.4457968543051,3.58957450803742,"19/09/1801","17:42:00",0,5,1801-09-19 17:42:00 +"1552806","Atlantis","10",51.4425450845039,3.59463539364189,"19/09/1801","19:36:00",0,5,1801-09-19 19:36:00 +"1552807","Atlantis","10",51.4441567413183,3.60005367420986,"19/09/1801","21:32:00",0,5,1801-09-19 21:32:00 +"1552808","Atlantis","10",51.447090309721,3.59577182918694,"19/09/1801","23:28:00",0,5,1801-09-19 23:28:00 +"1552809","Atlantis","10",51.4385658269576,3.59124963630829,"20/09/1801"," 1:22:00",8,121,1801-09-20 01:22:00 +"1552810","Atlantis","10",51.6690986812087,3.26875787262898,"20/09/1801"," 3:18:00",2,242,1801-09-20 03:18:00 +"1552811","Atlantis","10",51.6927783189821,3.22378886089847,"20/09/1801"," 5:14:00",3,54,1801-09-20 05:14:00 +"1552812","Atlantis","10",51.645804771685,3.14713836160861,"20/09/1801"," 7:08:00",2,36,1801-09-20 07:08:00 +"1552813","Atlantis","10",51.6374378545401,3.13225625028834,"20/09/1801"," 9:04:00",7,285,1801-09-20 09:04:00 +"1555385","Atlantis","10",51.6393622103091,3.10291450892761,"20/09/1801","10:58:00",0,0,1801-09-20 10:58:00 +"1555386","Atlantis","10",51.6427481339928,3.09245328405313,"20/09/1801","12:54:00",7,298,1801-09-20 12:54:00 +"1555387","Atlantis","10",51.6478532274701,3.0873692529602,"20/09/1801","14:50:00",5,301,1801-09-20 14:50:00 +"1555388","Atlantis","10",51.6268278803383,3.09525243391097,"20/09/1801","15:50:00",7,232,1801-09-20 15:50:00 +"1555389","Atlantis","10",51.6272949674968,3.07175986222364,"20/09/1801","16:44:00",6,313,1801-09-20 16:44:00 +"1555390","Atlantis","10",51.677005219867,3.14597554130386,"20/09/1801","18:40:00",4,136,1801-09-20 18:40:00 +"1555391","Atlantis","10",51.673920142578,3.23357144267764,"20/09/1801","20:34:00",6,109,1801-09-20 20:34:00 +"1555392","Atlantis","10",51.6958931647255,3.21720166679099,"20/09/1801","22:30:00",7,144,1801-09-20 22:30:00 +"1555393","Atlantis","10",51.6121485243998,3.10585340487864,"21/09/1801","00:26:00",6,267,1801-09-21 00:26:00 +"1555394","Atlantis","10",51.6081298023318,2.97478686106857,"21/09/1801"," 2:20:00",6,247,1801-09-21 02:20:00 +"1555395","Atlantis","10",51.6101169113438,2.97946308467071,"21/09/1801"," 2:20:00",6,247,1801-09-21 02:20:00 +"1555396","Atlantis","10",51.6058486282392,2.98607299769577,"21/09/1801"," 4:16:00",7,122,1801-09-21 04:16:00 +"1555397","Atlantis","10",51.5971814411571,2.94591727061663,"21/09/1801"," 6:10:00",8,74,1801-09-21 06:10:00 +"1555398","Atlantis","10",51.600600687827,3.00779928461835,"21/09/1801"," 8:06:00",5,68,1801-09-21 08:06:00 +"1555399","Atlantis","10",51.5715216041734,3.07609878732823,"21/09/1801","10:00:00",4,73,1801-09-21 10:00:00 +"1555400","Atlantis","10",51.5727340158257,3.07912814380229,"21/09/1801","11:56:00",7,149,1801-09-21 11:56:00 +"1555401","Atlantis","10",51.5285637688744,2.83894046649151,"21/09/1801","13:52:00",7,229,1801-09-21 13:52:00 +"1555402","Atlantis","10",51.5342896535024,2.86976746339165,"21/09/1801","15:46:00",7,228,1801-09-21 15:46:00 +"1555403","Atlantis","10",51.5512972140349,2.89446464026719,"21/09/1801","17:42:00",5,218,1801-09-21 17:42:00 +"1555404","Atlantis","10",51.5496101872013,2.88918378477078,"21/09/1801","19:38:00",7,53,1801-09-21 19:38:00 +"1555405","Atlantis","10",51.5692254377492,3.08719663170259,"21/09/1801","21:32:00",6,246,1801-09-21 21:32:00 +"1555406","Atlantis","10",51.6148937949184,3.07407225930411,"21/09/1801","23:28:00",7,66,1801-09-21 23:28:00 +"1555407","Atlantis","10",51.6100231290464,3.06001118563861,"21/09/1801","23:28:00",7,66,1801-09-21 23:28:00 +"1555408","Atlantis","10",51.547021713627,2.89482907915301,"22/09/1801"," 1:22:00",7,226,1801-09-22 01:22:00 +"1555409","Atlantis","10",51.5370821947339,2.87529989309702,"22/09/1801"," 5:12:00",5,96,1801-09-22 05:12:00 +"1555410","Atlantis","10",51.541101207267,2.85568756789807,"22/09/1801"," 7:08:00",7,46,1801-09-22 07:08:00 +"1555411","Atlantis","10",51.6432048825864,3.06129459817056,"22/09/1801"," 9:04:00",7,327,1801-09-22 09:04:00 +"1555412","Atlantis","10",51.64112977613,3.12847521154676,"22/09/1801","10:58:00",6,302,1801-09-22 10:58:00 +"1555413","Atlantis","10",51.6446539001246,3.14544970385451,"22/09/1801","12:54:00",6,183,1801-09-22 12:54:00 +"1555414","Atlantis","10",51.5335453653987,2.87610074411705,"22/09/1801","14:50:00",7,232,1801-09-22 14:50:00 +"1555415","Atlantis","10",51.5535119784218,2.84880464873929,"22/09/1801","20:34:00",7,308,1801-09-22 20:34:00 +"1555416","Atlantis","10",51.5474956538412,2.8758286859747,"22/09/1801","22:30:00",8,270,1801-09-22 22:30:00 +"1555417","Atlantis","10",51.5460690084323,2.89649960043747,"23/09/1801"," 2:20:00",7,62,1801-09-23 02:20:00 +"1555418","Atlantis","10",51.5514211131451,2.90268366619292,"23/09/1801"," 4:16:00",1,172,1801-09-23 04:16:00 +"1555419","Atlantis","10",51.5495581843243,2.85697844705079,"23/09/1801"," 6:10:00",6,282,1801-09-23 06:10:00 +"1555420","Atlantis","10",51.5253247259804,2.93190401869826,"23/09/1801"," 8:06:00",6,47,1801-09-23 08:06:00 +"1555421","Atlantis","10",51.517831816237,2.93616953798104,"23/09/1801"," 8:06:00",6,47,1801-09-23 08:06:00 +"1555422","Atlantis","10",51.5687327003279,3.03728956469335,"23/09/1801","10:02:00",5,235,1801-09-23 10:02:00 +"1555423","Atlantis","10",51.5480359526169,3.03731715998612,"23/09/1801","13:52:00",8,45,1801-09-23 13:52:00 +"1555424","Atlantis","10",51.459815652437,2.88228148337267,"23/09/1801","15:46:00",7,250,1801-09-23 15:46:00 +"1555425","Atlantis","10",51.3919987156806,2.60518348449562,"23/09/1801","17:42:00",7,249,1801-09-23 17:42:00 +"1555426","Atlantis","10",51.299196647421,2.3826450814344,"23/09/1801","19:36:00",6,209,1801-09-23 19:36:00 +"1555427","Atlantis","10",51.3144755225806,2.48406492707226,"23/09/1801","21:32:00",8,92,1801-09-23 21:32:00 +"1555428","Atlantis","10",51.4283873568,2.74755893425457,"23/09/1801","23:28:00",6,45,1801-09-23 23:28:00 +"1555429","Atlantis","10",51.5437486236542,2.96842689296883,"24/09/1801"," 1:22:00",6,16,1801-09-24 01:22:00 +"1555430","Atlantis","10",51.5686313771126,3.14876709777769,"24/09/1801"," 3:18:00",11,74,1801-09-24 03:18:00 +"1555431","Atlantis","10",51.4422775389431,3.60017206957005,"24/09/1801"," 5:14:00",5,306,1801-09-24 05:14:00 +"1555432","Atlantis","10",51.4456601491151,3.58971194659732,"24/09/1801"," 7:08:00",0,266,1801-09-24 07:08:00 +"1555433","Atlantis","10",51.4430409768759,3.58441896786355,"24/09/1801"," 9:04:00",0,266,1801-09-24 09:04:00 +"1555434","Atlantis","10",51.4415940989042,3.58721009844355,"24/09/1801","10:58:00",0,266,1801-09-24 10:58:00 +"1555435","Atlantis","10",51.4403935615346,3.58372482582368,"24/09/1801","12:54:00",0,266,1801-09-24 12:54:00 +"1555436","Atlantis","10",51.4464791745041,3.58346429321822,"24/09/1801","14:50:00",0,266,1801-09-24 14:50:00 +"1555437","Atlantis","10",51.44075881002,3.593833968224,"24/09/1801","16:44:00",0,266,1801-09-24 16:44:00 +"1558032","Atlantis","10",51.4416865218943,3.59898520351294,"26/09/1801","16:44:00",0,266,1801-09-26 16:44:00 +"1558033","Atlantis","10",51.444507561652,3.58986761561967,"26/09/1801","18:40:00",0,266,1801-09-26 18:40:00 +"1558034","Atlantis","10",51.4491718630446,3.59958758936636,"26/09/1801","20:34:00",0,266,1801-09-26 20:34:00 +"1558035","Atlantis","10",51.4469700975739,3.59976244255341,"26/09/1801","22:30:00",0,266,1801-09-26 22:30:00 +"1558036","Atlantis","10",51.4440934100258,3.58142909231503,"27/09/1801","00:26:00",0,266,1801-09-27 00:26:00 +"1558037","Atlantis","10",51.5682676910781,3.4050360801518,"27/09/1801"," 2:20:00",13,334,1801-09-27 02:20:00 +"1558038","Atlantis","10",51.9109818126173,3.0440084690284,"27/09/1801"," 4:16:00",13,334,1801-09-27 04:16:00 +"1558039","Atlantis","10",52.0014317679759,2.93187194291037,"27/09/1801"," 6:10:00",1,183,1801-09-27 06:10:00 +"1558040","Atlantis","10",52.0426699633361,2.91290088053234,"27/09/1801"," 8:06:00",3,98,1801-09-27 08:06:00 +"1558041","Atlantis","10",51.9973272840697,2.86476393927261,"27/09/1801","10:02:00",2,297,1801-09-27 10:02:00 +"1558042","Atlantis","10",51.9936305094389,2.95050260055531,"27/09/1801","11:56:00",6,100,1801-09-27 11:56:00 +"1558043","Atlantis","10",51.9916145632337,2.89220509236213,"27/09/1801","13:52:00",6,82,1801-09-27 13:52:00 +"1558044","Atlantis","10",52.0547769782222,2.74301726725139,"27/09/1801","15:46:00",6,304,1801-09-27 15:46:00 +"1558045","Atlantis","10",52.0614866272965,2.56911584057659,"27/09/1801","17:42:00",6,265,1801-09-27 17:42:00 +"1558046","Atlantis","10",52.1021421972322,2.45418863892369,"27/09/1801","19:38:00",5,167,1801-09-27 19:38:00 +"1558047","Atlantis","10",51.8635053773764,2.44897929977533,"27/09/1801","21:32:00",14,189,1801-09-27 21:32:00 +"1558048","Atlantis","10",51.5647645406644,2.26704707259778,"27/09/1801","23:28:00",6,227,1801-09-27 23:28:00 +"1558049","Atlantis","10",51.3949986578021,2.23858282744698,"28/09/1801"," 1:22:00",6,146,1801-09-28 01:22:00 +"1558050","Atlantis","10",51.4646490791421,2.33760085074324,"28/09/1801"," 3:18:00",7,49,1801-09-28 03:18:00 +"1558051","Atlantis","10",51.545809607177,2.52068897924666,"28/09/1801"," 5:14:00",6,215,1801-09-28 05:14:00 +"1558052","Atlantis","10",51.5283010063809,2.49850237113144,"28/09/1801"," 7:08:00",7,319,1801-09-28 07:08:00 +"1558053","Atlantis","10",51.5326686305585,2.51433237976302,"28/09/1801"," 9:04:00",3,91,1801-09-28 09:04:00 +"1558054","Atlantis","10",51.5231025944497,2.50100920245051,"28/09/1801","10:58:00",7,208,1801-09-28 10:58:00 +"1558055","Atlantis","10",51.3792545658685,2.29642233029846,"28/09/1801","12:54:00",5,201,1801-09-28 12:54:00 +"1558056","Atlantis","10",51.4779039047444,2.54954885991849,"28/09/1801","14:50:00",7,37,1801-09-28 14:50:00 +"1558057","Atlantis","10",51.5513259403594,2.63269043842983,"28/09/1801","16:44:00",9,60,1801-09-28 16:44:00 +"1558058","Atlantis","10",51.5519841404096,2.64188817282393,"28/09/1801","16:44:00",7,25,1801-09-28 16:44:00 +"1558059","Atlantis","10",51.5781261204942,2.67034391275607,"28/09/1801","18:40:00",6,212,1801-09-28 18:40:00 +"1558060","Atlantis","10",51.5803415251975,2.67682966098003,"28/09/1801","20:34:00",6,92,1801-09-28 20:34:00 +"1558061","Atlantis","10",51.5406331177237,2.57286354488134,"28/09/1801","22:30:00",7,198,1801-09-28 22:30:00 +"1558062","Atlantis","10",51.417292299855,2.46582766053081,"29/09/1801","00:26:00",4,39,1801-09-29 00:26:00 +"1558063","Atlantis","10",51.5286025949991,2.58255624916684,"29/09/1801"," 2:20:00",7,11,1801-09-29 02:20:00 +"1558064","Atlantis","10",51.5275327271684,2.57427951546479,"29/09/1801"," 4:16:00",5,194,1801-09-29 04:16:00 +"1558065","Atlantis","10",51.5050768146976,2.54326678625215,"29/09/1801"," 6:10:00",6,198,1801-09-29 06:10:00 +"1558066","Atlantis","10",51.5548413366494,2.57555594416708,"29/09/1801"," 8:06:00",6,18,1801-09-29 08:06:00 +"1558067","Atlantis","10",51.396479740486,2.53123941736296,"29/09/1801","10:02:00",6,34,1801-09-29 10:02:00 +"1558068","Atlantis","10",51.3657313061594,2.2747624752,"29/09/1801","13:52:00",12,301,1801-09-29 13:52:00 +"1558069","Atlantis","10",51.596758028449,1.91266153350473,"29/09/1801","15:46:00",7,16,1801-09-29 15:46:00 +"1558070","Atlantis","10",51.8183823845703,1.99119572775438,"29/09/1801","17:42:00",7,92,1801-09-29 17:42:00 +"1558071","Atlantis","10",51.7953515104749,2.14375905448478,"29/09/1801","19:38:00",12,105,1801-09-29 19:38:00 +"1558072","Atlantis","10",51.6509401507131,2.69030370800383,"29/09/1801","21:32:00",7,188,1801-09-29 21:32:00 +"1558073","Atlantis","10",51.4427872898132,2.58300456348806,"29/09/1801","23:28:00",2,233,1801-09-29 23:28:00 +"1558074","Atlantis","10",51.3148875146508,2.42021662644949,"30/09/1801"," 1:22:00",6,233,1801-09-30 01:22:00 +"1558075","Atlantis","10",51.3360548409061,2.51276356823835,"30/09/1801"," 3:18:00",7,34,1801-09-30 03:18:00 +"1558076","Atlantis","10",51.4402426631981,2.79793558369856,"30/09/1801"," 5:12:00",7,65,1801-09-30 05:12:00 +"1558077","Atlantis","10",51.5705081450548,3.08113315943349,"30/09/1801"," 7:08:00",7,7,1801-09-30 07:08:00 +"1558078","Atlantis","10",51.5700285940524,3.0875722193867,"30/09/1801"," 9:04:00",6,84,1801-09-30 09:04:00 +"1558079","Atlantis","10",51.5528322362537,3.03265564845782,"30/09/1801","10:28:00",5,49,1801-09-30 10:28:00 +"1558080","Atlantis","10",51.569113122283,3.09581196783483,"30/09/1801","10:58:00",7,195,1801-09-30 10:58:00 +"1558081","Atlantis","10",51.5629528832883,3.06528747584112,"30/09/1801","12:34:00",7,231,1801-09-30 12:34:00 +"1558082","Atlantis","10",51.5386679790528,3.01583193154726,"30/09/1801","12:54:00",6,229,1801-09-30 12:54:00 +"1558083","Atlantis","10",51.5566054825126,3.05772055265307,"30/09/1801","14:50:00",7,46,1801-09-30 14:50:00 +"1558084","Atlantis","10",51.5746130309128,3.06660910889227,"30/09/1801","15:30:00",6,267,1801-09-30 15:30:00 +"1558085","Atlantis","10",51.5547350416905,3.0379128654208,"30/09/1801","16:44:00",6,134,1801-09-30 16:44:00 +"1558086","Atlantis","10",51.5738008662565,2.91443522435706,"30/09/1801","18:40:00",7,312,1801-09-30 18:40:00 +"1558087","Atlantis","10",51.5816079751374,2.89790764666535,"30/09/1801","20:34:00",6,133,1801-09-30 20:34:00 +"1558088","Atlantis","10",51.5904997833855,2.95418330026325,"30/09/1801","22:30:00",5,42,1801-09-30 22:30:00 +"1558089","Atlantis","10",51.5388710420597,3.03839973628055,"01/10/1801","00:24:00",2,215,1801-10-01 00:24:00 +"1558090","Atlantis","10",51.5406176664173,3.07180915182456,"01/10/1801"," 2:20:00",11,101,1801-10-01 02:20:00 +"1558091","Atlantis","10",51.442142100336,3.58598931783345,"01/10/1801"," 4:16:00",0,296,1801-10-01 04:16:00 +"1558092","Atlantis","10",51.448800364316,3.59860111930314,"01/10/1801"," 6:10:00",0,93,1801-10-01 06:10:00 +"1558093","Atlantis","10",51.4408553106524,3.59381541414093,"01/10/1801"," 8:06:00",0,93,1801-10-01 08:06:00 +"1558094","Atlantis","10",51.4453299607523,3.58830646265205,"01/10/1801","10:02:00",0,93,1801-10-01 10:02:00 +"1558095","Atlantis","10",51.4477342711506,3.58414604367595,"01/10/1801","11:56:00",0,93,1801-10-01 11:56:00 +"1558096","Atlantis","10",51.4480933251628,3.58359541268926,"01/10/1801","13:52:00",0,93,1801-10-01 13:52:00 +"1558097","Atlantis","10",51.4471614910383,3.59122742063273,"01/10/1801","15:46:00",0,93,1801-10-01 15:46:00 +"1493719","Atlantis","1000",53.0434159252392,4.86039992010314,"06/09/1801"," 7:52:00",0,0,1801-09-06 07:52:00 +"1493720","Atlantis","1000",53.0391928949621,4.85573100547399,"06/09/1801"," 8:02:00",0,0,1801-09-06 08:02:00 +"1493721","Atlantis","1000",52.8786157848206,4.62145266381185,"06/09/1801"," 9:56:00",7,203,1801-09-06 09:56:00 +"1493722","Atlantis","1000",52.6651118033011,4.4838202519454,"06/09/1801","11:52:00",8,216,1801-09-06 11:52:00 +"1493723","Atlantis","1000",52.4562779938714,4.2572144849645,"06/09/1801","13:48:00",6,209,1801-09-06 13:48:00 +"1493724","Atlantis","1000",52.1225659727901,4.0045443233531,"06/09/1801","17:38:00",6,205,1801-09-06 17:38:00 +"1493725","Atlantis","1000",52.0587373466915,3.89711018315144,"06/09/1801","19:32:00",6,257,1801-09-06 19:32:00 +"1493726","Atlantis","1000",52.0879695585212,3.8076382340109,"06/09/1801","21:28:00",6,79,1801-09-06 21:28:00 +"1493727","Atlantis","1000",52.0943996549281,3.92925377265457,"06/09/1801","23:22:00",6,279,1801-09-06 23:22:00 +"1493728","Atlantis","1000",52.0927054601139,3.96246439004596,"07/09/1801"," 1:18:00",6,293,1801-09-07 01:18:00 +"1493729","Atlantis","1000",52.0253866966981,3.71335315156542,"07/09/1801"," 3:14:00",6,201,1801-09-07 03:14:00 +"1493730","Atlantis","1000",51.9032305564419,3.54204537094012,"07/09/1801"," 5:08:00",6,266,1801-09-07 05:08:00 +"1493731","Atlantis","1000",51.8173588924212,3.43941855607089,"07/09/1801"," 7:04:00",5,36,1801-09-07 07:04:00 +"1493732","Atlantis","1000",51.8248535044868,3.39852710428182,"07/09/1801"," 9:00:00",5,323,1801-09-07 09:00:00 +"1493733","Atlantis","1000",51.8922090479923,3.52017652205657,"07/09/1801","10:54:00",5,59,1801-09-07 10:54:00 +"1493734","Atlantis","1000",51.9213675553459,3.6420488694869,"07/09/1801","12:50:00",5,69,1801-09-07 12:50:00 +"1493735","Atlantis","1000",51.8913545836741,3.52167688786518,"07/09/1801","14:44:00",7,216,1801-09-07 14:44:00 +"1493736","Atlantis","1000",51.8116169474665,3.4032082766816,"07/09/1801","16:40:00",2,333,1801-09-07 16:40:00 +"1493737","Atlantis","1000",51.7950095808492,3.38747696138639,"07/09/1801","18:34:00",5,116,1801-09-07 18:34:00 +"1493738","Atlantis","1000",51.797432103775,3.38437063988485,"07/09/1801","18:34:00",5,116,1801-09-07 18:34:00 +"1493739","Atlantis","1000",51.7787881947774,3.35508410959877,"07/09/1801","20:30:00",2,237,1801-09-07 20:30:00 +"1493740","Atlantis","1000",51.8792570095169,3.46880157367978,"07/09/1801","22:26:00",4,64,1801-09-07 22:26:00 +"1493741","Atlantis","1000",51.9029700058745,3.58492405588459,"08/09/1801","00:20:00",5,64,1801-09-08 00:20:00 +"1493742","Atlantis","1000",51.9081234872513,3.57487422392703,"08/09/1801"," 2:16:00",7,212,1801-09-08 02:16:00 +"1493743","Atlantis","1000",51.8434572159438,3.43192138470989,"08/09/1801"," 4:10:00",5,162,1801-09-08 04:10:00 +"1493744","Atlantis","1000",51.8077329211943,3.4345757668959,"08/09/1801"," 6:06:00",5,31,1801-09-08 06:06:00 +"1493745","Atlantis","1000",51.8144763873336,3.36309118957631,"08/09/1801"," 8:02:00",0,328,1801-09-08 08:02:00 +"1493746","Atlantis","1000",51.8695243590521,3.45338490560465,"08/09/1801"," 9:56:00",4,68,1801-09-08 09:56:00 +"1493747","Atlantis","1000",51.9439359065038,3.64686058025341,"08/09/1801","11:52:00",5,42,1801-09-08 11:52:00 +"1493748","Atlantis","1000",52.0895222066161,3.77783132208604,"08/09/1801","13:46:00",5,24,1801-09-08 13:46:00 +"1493749","Atlantis","1000",52.074628931968,3.81902042715531,"08/09/1801","15:42:00",5,271,1801-09-08 15:42:00 +"1493750","Atlantis","1000",52.0505141853681,3.77911824222375,"08/09/1801","17:38:00",4,323,1801-09-08 17:38:00 +"1493751","Atlantis","1000",51.9478016214454,3.6316532344101,"08/09/1801","19:32:00",6,257,1801-09-08 19:32:00 +"1493752","Atlantis","1000",51.9255596830077,3.63964854777604,"08/09/1801","21:28:00",5,86,1801-09-08 21:28:00 +"1493753","Atlantis","1000",52.0790483078011,3.76505983921513,"08/09/1801","23:22:00",5,42,1801-09-08 23:22:00 +"1493754","Atlantis","1000",52.1189892186206,3.91242259835266,"09/09/1801"," 1:18:00",6,18,1801-09-09 01:18:00 +"1493755","Atlantis","1000",52.1977965749567,4.03307513936423,"09/09/1801"," 3:14:00",6,256,1801-09-09 03:14:00 +"1498316","Atlantis","1000",52.1377864535069,3.95042461542972,"09/09/1801"," 5:08:00",3,257,1801-09-09 05:08:00 +"1498317","Atlantis","1000",52.1039771344992,3.90317872717883,"09/09/1801"," 7:04:00",5,107,1801-09-09 07:04:00 +"1498318","Atlantis","1000",52.058072812485,3.72920788873173,"09/09/1801"," 9:00:00",6,215,1801-09-09 09:00:00 +"1498319","Atlantis","1000",51.9438128281957,3.57122923247237,"09/09/1801","10:54:00",5,232,1801-09-09 10:54:00 +"1498320","Atlantis","1000",51.8733214770723,3.60170960045699,"09/09/1801","12:50:00",6,53,1801-09-09 12:50:00 +"1498321","Atlantis","1000",51.9068747723,3.76527937569376,"09/09/1801","14:44:00",0,237,1801-09-09 14:44:00 +"1498322","Atlantis","1000",51.9079607464219,3.75440723025799,"09/09/1801","16:40:00",4,85,1801-09-09 16:40:00 +"1498323","Atlantis","1000",51.9230772269531,3.76959852884989,"09/09/1801","18:36:00",4,236,1801-09-09 18:36:00 +"1498324","Atlantis","1000",51.9160086156051,3.73847772618663,"09/09/1801","20:30:00",6,265,1801-09-09 20:30:00 +"1498325","Atlantis","1000",51.9140625757859,3.73183603986818,"09/09/1801","22:26:00",5,261,1801-09-09 22:26:00 +"1498326","Atlantis","1000",51.9216730084303,3.78951853413507,"10/09/1801","00:20:00",4,255,1801-09-10 00:20:00 +"1498327","Atlantis","1000",52.0070177473454,3.74435810361244,"10/09/1801"," 2:16:00",6,50,1801-09-10 02:16:00 +"1498328","Atlantis","1000",52.0857335051731,3.95902178882435,"10/09/1801"," 4:10:00",6,74,1801-09-10 04:10:00 +"1498329","Atlantis","1000",52.1020263903136,4.26419685279857,"10/09/1801"," 6:06:00",0,52,1801-09-10 06:06:00 +"1498330","Atlantis","1000",52.099411649066,4.26755675434135,"10/09/1801"," 8:02:00",0,52,1801-09-10 08:02:00 +"1498331","Atlantis","1000",52.097921293023,4.26781877661776,"10/09/1801"," 9:56:00",0,52,1801-09-10 09:56:00 +"1498332","Atlantis","1000",52.099881211027,4.26120797901601,"10/09/1801","11:52:00",0,52,1801-09-10 11:52:00 +"1498333","Atlantis","1000",52.105552203936,4.26040909489803,"10/09/1801","13:46:00",0,52,1801-09-10 13:46:00 +"1498334","Atlantis","1000",52.1015861303303,4.27405107150506,"10/09/1801","15:42:00",0,52,1801-09-10 15:42:00 +"1498335","Atlantis","1000",52.0990175321782,4.26158876291197,"10/09/1801","17:38:00",0,52,1801-09-10 17:38:00 +"1498381","Atlantis","1000",52.1057107336088,4.26755282586813,"14/09/1801","21:28:00",0,254,1801-09-14 21:28:00 +"1498382","Atlantis","1000",52.1023270196528,4.26463423678093,"14/09/1801","23:22:00",0,254,1801-09-14 23:22:00 +"1498383","Atlantis","1000",52.0965018301774,4.2636389182033,"15/09/1801"," 1:18:00",0,254,1801-09-15 01:18:00 +"1498384","Atlantis","1000",52.1042713696547,4.25777227747161,"15/09/1801"," 3:14:00",0,254,1801-09-15 03:14:00 +"1498385","Atlantis","1000",52.1048047531317,4.27222119367122,"15/09/1801"," 5:08:00",0,254,1801-09-15 05:08:00 +"1498386","Atlantis","1000",52.1008929254571,4.26494790281076,"15/09/1801"," 7:04:00",0,254,1801-09-15 07:04:00 +"1498387","Atlantis","1000",52.1950036195507,4.15780540184956,"15/09/1801"," 9:00:00",6,330,1801-09-15 09:00:00 +"1498388","Atlantis","1000",52.1849902982879,4.13210351084359,"15/09/1801","14:44:00",6,72,1801-09-15 14:44:00 +"1498389","Atlantis","1000",52.1666793614435,4.20295361393783,"15/09/1801","16:40:00",6,339,1801-09-15 16:40:00 +"1502939","Atlantis","1000",52.1961251398046,4.29312199095264,"15/09/1801","18:34:00",6,39,1801-09-15 18:34:00 +"1502940","Atlantis","1000",52.3458119147778,4.43052395474911,"15/09/1801","20:30:00",6,23,1801-09-15 20:30:00 +"1502941","Atlantis","1000",52.3384699962661,4.23194770198036,"15/09/1801","22:26:00",7,180,1801-09-15 22:26:00 +"1502942","Atlantis","1000",52.2380342839002,4.22860902754776,"16/09/1801","00:20:00",5,117,1801-09-16 00:20:00 +"1502943","Atlantis","1000",52.2273913278659,4.16403655192349,"16/09/1801"," 2:16:00",6,280,1801-09-16 02:16:00 +"1502944","Atlantis","1000",52.2230496009174,4.06598555628676,"16/09/1801"," 4:10:00",6,36,1801-09-16 04:10:00 +"1502945","Atlantis","1000",52.3393740825071,4.19296547806729,"16/09/1801"," 6:06:00",6,115,1801-09-16 06:06:00 +"1502946","Atlantis","1000",52.3513445997867,4.22717295426968,"16/09/1801"," 8:02:00",6,292,1801-09-16 08:02:00 +"1502947","Atlantis","1000",52.3566199783008,4.2469436198147,"16/09/1801"," 9:56:00",4,252,1801-09-16 09:56:00 +"1502948","Atlantis","1000",52.2465200651605,4.33544034792855,"16/09/1801","11:52:00",7,186,1801-09-16 11:52:00 +"1502949","Atlantis","1000",52.1827650199635,4.15517145046592,"16/09/1801","13:48:00",6,132,1801-09-16 13:48:00 +"1502950","Atlantis","1000",52.1740664670044,4.1620262981439,"16/09/1801","15:42:00",6,70,1801-09-16 15:42:00 +"1502951","Atlantis","1000",52.3210810701735,4.26800249421597,"16/09/1801","17:38:00",4,332,1801-09-16 17:38:00 +"1502952","Atlantis","1000",52.4992762543284,4.28171220722515,"16/09/1801","19:32:00",7,11,1801-09-16 19:32:00 +"1502953","Atlantis","1000",52.6744520382681,4.38678845097031,"16/09/1801","21:28:00",6,37,1801-09-16 21:28:00 +"1502954","Atlantis","1000",52.8052060831105,4.51107283907197,"16/09/1801","23:22:00",7,250,1801-09-16 23:22:00 +"1502955","Atlantis","1000",52.8370997888916,4.52895363173447,"17/09/1801"," 1:18:00",5,109,1801-09-17 01:18:00 +"1502956","Atlantis","1000",52.8020524877235,4.53773292665836,"17/09/1801"," 3:14:00",6,161,1801-09-17 03:14:00 +"1502957","Atlantis","1000",52.7444767662696,4.56012132533546,"17/09/1801"," 5:08:00",6,355,1801-09-17 05:08:00 +"1502958","Atlantis","1000",53.0344044417758,4.51360514394194,"17/09/1801"," 9:00:00",5,309,1801-09-17 09:00:00 +"1502959","Atlantis","1000",52.9342191733611,4.53843784791697,"17/09/1801","12:50:00",2,67,1801-09-17 12:50:00 +"1502960","Atlantis","1000",52.9199692168469,4.53248457246367,"17/09/1801","14:46:00",5,185,1801-09-17 14:46:00 +"1502961","Atlantis","1000",52.834716565839,4.46923180126306,"17/09/1801","16:40:00",5,101,1801-09-17 16:40:00 +"1502962","Atlantis","1000",52.8352378503825,4.50703666216135,"17/09/1801","18:36:00",8,64,1801-09-17 18:36:00 +"1502963","Atlantis","1000",52.8895322747892,4.50618051727489,"17/09/1801","22:26:00",5,293,1801-09-17 22:26:00 +"1502964","Atlantis","1000",52.9758086879603,4.36096535498649,"18/09/1801","00:20:00",5,350,1801-09-18 00:20:00 +"1502965","Atlantis","1000",52.8957018431895,4.45537037542183,"18/09/1801"," 2:16:00",5,105,1801-09-18 02:16:00 +"1502966","Atlantis","1000",52.8963877414763,4.55304866882786,"18/09/1801"," 4:10:00",5,101,1801-09-18 04:10:00 +"1502967","Atlantis","1000",52.9716838433202,4.78857516254764,"18/09/1801"," 6:06:00",7,127,1801-09-18 06:06:00 +"1502968","Atlantis","1000",52.9567094779066,4.78081506850477,"18/09/1801"," 8:02:00",0,0,1801-09-18 08:02:00 +"1502969","Atlantis","1000",53.0464929563478,4.85759623927623,"18/09/1801"," 9:56:00",0,22,1801-09-18 09:56:00 +"1502970","Atlantis","1000",53.0391089843512,4.85734033415653,"18/09/1801","11:52:00",0,22,1801-09-18 11:52:00 +"1502971","Atlantis","1000",53.0423311868235,4.86082503077201,"18/09/1801","13:46:00",0,22,1801-09-18 13:46:00 +"1502972","Atlantis","1000",53.0387893150835,4.85287608387973,"18/09/1801","15:42:00",0,22,1801-09-18 15:42:00 +"1502973","Atlantis","1000",53.0440184082552,4.84544364344422,"18/09/1801","17:38:00",0,22,1801-09-18 17:38:00 +"1507626","Atlantis","1000",53.0383256842936,4.85751508723572,"26/09/1801","11:52:00",0,22,1801-09-26 11:52:00 +"1507627","Atlantis","1000",53.0435960803526,4.85607188084256,"26/09/1801","13:48:00",0,22,1801-09-26 13:48:00 +"1507628","Atlantis","1000",53.0407864031307,4.86148977910262,"26/09/1801","15:42:00",0,22,1801-09-26 15:42:00 +"1507629","Atlantis","1000",53.0416237840611,4.85779454235081,"26/09/1801","17:38:00",0,22,1801-09-26 17:38:00 +"1507630","Atlantis","1000",53.0471266337736,4.86129902815074,"26/09/1801","19:32:00",0,22,1801-09-26 19:32:00 +"1507631","Atlantis","1000",53.0405591691723,4.85337611228786,"26/09/1801","21:28:00",0,22,1801-09-26 21:28:00 +"1507632","Atlantis","1000",53.0454804953258,4.68400068532675,"26/09/1801","23:24:00",8,286,1801-09-26 23:24:00 +"1507633","Atlantis","1000",52.9214232642087,4.53928504845407,"27/09/1801"," 1:18:00",6,204,1801-09-27 01:18:00 +"1507634","Atlantis","1000",52.9411627344843,4.4911656720927,"27/09/1801"," 3:14:00",5,298,1801-09-27 03:14:00 +"1507635","Atlantis","1000",52.9871179323704,4.523928632644,"27/09/1801"," 5:08:00",5,315,1801-09-27 05:08:00 +"1507636","Atlantis","1000",53.0175903242375,4.59168570462428,"27/09/1801"," 7:04:00",11,63,1801-09-27 07:04:00 +"1507637","Atlantis","1000",52.972059805667,4.50636399603263,"27/09/1801"," 8:58:00",3,319,1801-09-27 08:58:00 +"1507638","Atlantis","1000",53.0186845638072,4.56009708560072,"27/09/1801","10:54:00",5,149,1801-09-27 10:54:00 +"1507639","Atlantis","1000",52.8681100822063,4.65038123203255,"27/09/1801","12:50:00",1,118,1801-09-27 12:50:00 +"1507640","Atlantis","1000",52.7593794604233,4.58236632394884,"27/09/1801","14:44:00",5,287,1801-09-27 14:44:00 +"1507641","Atlantis","1000",52.8377421631296,4.43028642858192,"27/09/1801","16:40:00",6,83,1801-09-27 16:40:00 +"1507642","Atlantis","1000",52.8739825280649,4.52850564993545,"27/09/1801","18:36:00",6,3,1801-09-27 18:36:00 +"1507643","Atlantis","1000",53.0189731152924,4.53151862071268,"27/09/1801","20:30:00",5,81,1801-09-27 20:30:00 +"1507644","Atlantis","1000",53.0651295589269,4.41998828342091,"27/09/1801","22:26:00",5,286,1801-09-27 22:26:00 +"1507645","Atlantis","1000",53.003674019353,4.54597803249024,"28/09/1801","00:20:00",6,128,1801-09-28 00:20:00 +"1507646","Atlantis","1000",52.8373987062164,4.48244092096761,"28/09/1801"," 2:16:00",6,188,1801-09-28 02:16:00 +"1512204","Atlantis","1000",52.7211464689099,4.45581009457726,"28/09/1801"," 4:10:00",6,55,1801-09-28 04:10:00 +"1512205","Atlantis","1000",52.8848246160885,4.56567361652572,"28/09/1801"," 6:06:00",6,17,1801-09-28 06:06:00 +"1512206","Atlantis","1000",52.9685201412635,4.54201477331854,"28/09/1801"," 8:02:00",5,292,1801-09-28 08:02:00 +"1512207","Atlantis","1000",53.0413363377377,4.6127754547447,"28/09/1801"," 9:56:00",2,28,1801-09-28 09:56:00 +"1512208","Atlantis","1000",53.0015929468656,4.57043749057129,"28/09/1801","11:52:00",4,269,1801-09-28 11:52:00 +"1512209","Atlantis","1000",52.9699303423516,4.38927855264861,"28/09/1801","13:48:00",6,263,1801-09-28 13:48:00 +"1512210","Atlantis","1000",53.0942929543839,4.29571476837154,"28/09/1801","15:42:00",6,325,1801-09-28 15:42:00 +"1512211","Atlantis","1000",53.2568992878282,4.22876921610255,"28/09/1801","17:38:00",7,12,1801-09-28 17:38:00 +"1512212","Atlantis","1000",53.4367072860659,4.28098407425545,"28/09/1801","19:32:00",6,358,1801-09-28 19:32:00 +"1512213","Atlantis","1000",53.4859683703231,4.25668919563666,"28/09/1801","21:28:00",5,180,1801-09-28 21:28:00 +"1512214","Atlantis","1000",53.4882732125446,4.27515190379415,"28/09/1801","23:22:00",6,110,1801-09-28 23:22:00 +"1512215","Atlantis","1000",53.4636889837482,4.52624924775306,"29/09/1801"," 1:18:00",5,58,1801-09-29 01:18:00 +"1512216","Atlantis","1000",53.4951235034154,4.30082263695169,"29/09/1801"," 3:14:00",5,282,1801-09-29 03:14:00 +"1512217","Atlantis","1000",53.4089845649297,4.4895406128522,"29/09/1801"," 5:08:00",6,141,1801-09-29 05:08:00 +"1512218","Atlantis","1000",53.4167837433522,4.44760875632614,"29/09/1801"," 7:04:00",5,337,1801-09-29 07:04:00 +"1512219","Atlantis","1000",53.4141744814115,4.43579513007589,"29/09/1801"," 8:58:00",7,111,1801-09-29 08:58:00 +"1512220","Atlantis","1000",53.3896926419199,4.40086957068741,"29/09/1801","12:50:00",4,147,1801-09-29 12:50:00 +"1512221","Atlantis","1000",53.382714761775,4.49920261679031,"29/09/1801","14:44:00",0,164,1801-09-29 14:44:00 +"1512222","Atlantis","1000",53.3952453846992,4.51067131389771,"29/09/1801","16:40:00",5,112,1801-09-29 16:40:00 +"1512223","Atlantis","1000",53.4176659024577,4.507446687866,"29/09/1801","18:36:00",5,289,1801-09-29 18:36:00 +"1512224","Atlantis","1000",53.3801557621583,4.53723492761236,"29/09/1801","20:30:00",6,91,1801-09-29 20:30:00 +"1512225","Atlantis","1000",53.4352834003186,4.64812511865422,"29/09/1801","22:26:00",6,305,1801-09-29 22:26:00 +"1512226","Atlantis","1000",53.3494218198801,4.46770379958768,"30/09/1801","00:20:00",1,282,1801-09-30 00:20:00 +"1512227","Atlantis","1000",53.3424762216341,4.34697645178996,"30/09/1801"," 2:16:00",5,76,1801-09-30 02:16:00 +"1512228","Atlantis","1000",53.3370280859522,4.42569616551325,"30/09/1801"," 4:10:00",5,161,1801-09-30 04:10:00 +"1512229","Atlantis","1000",53.3558751911884,4.47312990063894,"30/09/1801"," 6:06:00",6,28,1801-09-30 06:06:00 +"1512230","Atlantis","1000",53.40505229514,4.31221120379679,"30/09/1801"," 8:02:00",6,138,1801-09-30 08:02:00 +"1512231","Atlantis","1000",53.3898518887577,4.33386715704575,"30/09/1801"," 9:56:00",5,280,1801-09-30 09:56:00 +"1512232","Atlantis","1000",53.3874105561376,4.30660998167191,"30/09/1801","11:52:00",6,281,1801-09-30 11:52:00 +"1512233","Atlantis","1000",53.3878246128513,4.25705507229734,"30/09/1801","13:46:00",6,133,1801-09-30 13:46:00 +"1512234","Atlantis","1000",53.3582406851947,4.25134513336979,"30/09/1801","15:42:00",6,175,1801-09-30 15:42:00 +"1512235","Atlantis","1000",53.4051861366234,4.2894739327291,"30/09/1801","17:38:00",5,180,1801-09-30 17:38:00 +"1512236","Atlantis","1000",53.3936309569199,4.26617501014751,"30/09/1801","19:32:00",6,356,1801-09-30 19:32:00 +"1512237","Atlantis","1000",53.355841866516,4.39583778709732,"30/09/1801","21:28:00",5,159,1801-09-30 21:28:00 +"1512238","Atlantis","1000",53.2234248305834,4.38129454640485,"30/09/1801","23:24:00",6,181,1801-09-30 23:24:00 +"1512239","Atlantis","1000",53.0655651656962,4.49305760519858,"01/10/1801"," 1:18:00",6,197,1801-10-01 01:18:00 +"1512240","Atlantis","1000",52.92363798668,4.5444898406826,"01/10/1801"," 3:14:00",6,192,1801-10-01 03:14:00 +"1512241","Atlantis","1000",52.8624327646787,4.49109413930867,"01/10/1801"," 5:08:00",6,315,1801-10-01 05:08:00 +"1512242","Atlantis","1000",52.917895494231,4.66352581830509,"01/10/1801"," 7:04:00",10,35,1801-10-01 07:04:00 +"1512243","Atlantis","1000",52.9525603733738,4.77155911312625,"01/10/1801"," 8:58:00",0,164,1801-10-01 08:58:00 +"1512244","Atlantis","1000",53.0458838200402,4.84651307640691,"01/10/1801","10:54:00",0,21,1801-10-01 10:54:00 +"1512245","Atlantis","1000",53.0423886302924,4.85602574569732,"01/10/1801","12:50:00",0,21,1801-10-01 12:50:00 +"1512246","Atlantis","1000",53.0422707421188,4.85222774203494,"01/10/1801","14:44:00",0,21,1801-10-01 14:44:00 +"1512247","Atlantis","1000",53.0387854943001,4.86357425378263,"01/10/1801","16:40:00",0,21,1801-10-01 16:40:00 +"1512248","Atlantis","1000",53.0384856555746,4.85492216722667,"01/10/1801","18:34:00",0,21,1801-10-01 18:34:00 +"755434","Atlantis","1004",53.0410637151897,4.858329753072,"04/05/1800","22:22:00",0,36,1800-05-04 22:22:00 +"755435","Atlantis","1004",53.0398865111154,4.85689350460842,"04/05/1800","23:54:00",0,36,1800-05-04 23:54:00 +"755436","Atlantis","1004",53.0438210933241,4.8530598966144,"05/05/1800"," 2:06:00",0,36,1800-05-05 02:06:00 +"755437","Atlantis","1004",53.0425317819538,4.85701781604625,"05/05/1800"," 4:04:00",0,36,1800-05-05 04:04:00 +"755438","Atlantis","1004",53.0468528502602,4.85965077394806,"05/05/1800"," 5:34:00",0,36,1800-05-05 05:34:00 +"755439","Atlantis","1004",52.895218875133,4.57177000698354,"05/05/1800"," 9:30:00",13,253,1800-05-05 09:30:00 +"755440","Atlantis","1004",52.8443452505716,4.32842034869175,"05/05/1800","11:16:00",7,36,1800-05-05 11:16:00 +"755441","Atlantis","1004",52.8427088121311,4.27484300872404,"05/05/1800","13:44:00",7,103,1800-05-05 13:44:00 +"755442","Atlantis","1004",52.8156396426102,4.28638469165098,"05/05/1800","15:08:00",7,198,1800-05-05 15:08:00 +"755443","Atlantis","1004",52.9883903364171,4.28869952339493,"05/05/1800","17:36:00",7,7,1800-05-05 17:36:00 +"755444","Atlantis","1004",52.9975110820504,4.22593754480313,"05/05/1800","19:04:00",6,196,1800-05-05 19:04:00 +"755445","Atlantis","1004",53.0120883995886,4.18397610693797,"05/05/1800","21:20:00",7,160,1800-05-05 21:20:00 +"755446","Atlantis","1004",52.8212061220631,4.1863530101832,"05/05/1800","23:12:00",7,200,1800-05-05 23:12:00 +"755447","Atlantis","1004",52.7768871428445,4.28019045575429,"06/05/1800","00:46:00",7,143,1800-05-06 00:46:00 +"755448","Atlantis","1004",52.6479988184483,4.25120409695804,"06/05/1800"," 3:08:00",7,20,1800-05-06 03:08:00 +"755449","Atlantis","1004",52.7312412026022,4.27164341187105,"06/05/1800"," 4:46:00",7,184,1800-05-06 04:46:00 +"755450","Atlantis","1004",52.7746488657403,4.23060119117331,"06/05/1800"," 6:36:00",7,276,1800-05-06 06:36:00 +"755451","Atlantis","1004",52.8115275521972,4.20150245377421,"06/05/1800"," 8:42:00",7,304,1800-05-06 08:42:00 +"755452","Atlantis","1004",52.9582336360016,4.13177466383577,"06/05/1800","10:32:00",1,33,1800-05-06 10:32:00 +"755453","Atlantis","1004",52.8105765686468,4.02608461197093,"06/05/1800","12:36:00",6,185,1800-05-06 12:36:00 +"755454","Atlantis","1004",52.6277495644484,3.92510407305416,"06/05/1800","14:20:00",7,195,1800-05-06 14:20:00 +"755455","Atlantis","1004",52.6563245558548,3.9138392011011,"06/05/1800","16:08:00",7,283,1800-05-06 16:08:00 +"755456","Atlantis","1004",52.6820222244021,3.96188473174162,"06/05/1800","18:14:00",7,24,1800-05-06 18:14:00 +"755457","Atlantis","1004",52.7137377939639,3.97940136934072,"06/05/1800","20:18:00",7,248,1800-05-06 20:18:00 +"755458","Atlantis","1004",52.6831122465655,3.94130963684246,"06/05/1800","21:52:00",7,21,1800-05-06 21:52:00 +"755459","Atlantis","1004",52.5836684614932,3.81500060566608,"07/05/1800","00:18:00",7,272,1800-05-07 00:18:00 +"755460","Atlantis","1004",52.6078598592002,3.79763066495769,"07/05/1800"," 1:58:00",7,3,1800-05-07 01:58:00 +"755461","Atlantis","1004",52.6861749686715,3.94431631326955,"07/05/1800"," 3:42:00",8,235,1800-05-07 03:42:00 +"755462","Atlantis","1004",52.781947661784,4.0400970756039,"07/05/1800"," 5:36:00",5,41,1800-05-07 05:36:00 +"755463","Atlantis","1004",52.8244807097595,4.0866260997504,"07/05/1800"," 7:46:00",7,331,1800-05-07 07:46:00 +"755464","Atlantis","1004",52.8752356817019,4.05530407363828,"07/05/1800"," 9:34:00",6,235,1800-05-07 09:34:00 +"755465","Atlantis","1004",52.7391884190724,4.01452104163077,"07/05/1800","11:16:00",7,186,1800-05-07 11:16:00 +"755466","Atlantis","1004",52.6217233637175,3.97294506056048,"07/05/1800","13:34:00",8,207,1800-05-07 13:34:00 +"755467","Atlantis","1004",52.572891259009,3.90145046227984,"07/05/1800","15:32:00",7,20,1800-05-07 15:32:00 +"759373","Atlantis","1004",52.694721649827,3.93952235780098,"07/05/1800","17:22:00",0,315,1800-05-07 17:22:00 +"759374","Atlantis","1004",52.7403297130312,3.86636112119071,"08/05/1800"," 1:12:00",7,330,1800-05-08 01:12:00 +"759375","Atlantis","1004",52.7663727129912,3.9763517176602,"08/05/1800"," 3:06:00",7,234,1800-05-08 03:06:00 +"759376","Atlantis","1004",52.7820742196948,3.96445956499688,"08/05/1800"," 4:50:00",7,92,1800-05-08 04:50:00 +"759377","Atlantis","1004",52.7694344382365,3.91430960608367,"08/05/1800"," 6:32:00",7,98,1800-05-08 06:32:00 +"759378","Atlantis","1004",52.765488364893,3.99230497984216,"08/05/1800"," 8:58:00",8,66,1800-05-08 08:58:00 +"759379","Atlantis","1004",52.7545567548112,3.90839323052764,"08/05/1800","10:48:00",7,291,1800-05-08 10:48:00 +"759380","Atlantis","1004",52.8990048942389,3.90936921841651,"08/05/1800","12:28:00",7,129,1800-05-08 12:28:00 +"759381","Atlantis","1004",52.9940440721009,3.98881037372164,"08/05/1800","14:10:00",8,188,1800-05-08 14:10:00 +"759382","Atlantis","1004",52.813348988255,3.93535076831654,"08/05/1800","16:20:00",7,347,1800-05-08 16:20:00 +"759383","Atlantis","1004",52.8572157871951,3.93399148480594,"08/05/1800","20:00:00",7,209,1800-05-08 20:00:00 +"759384","Atlantis","1004",52.791672152814,3.89442145360261,"08/05/1800","22:16:00",7,2,1800-05-08 22:16:00 +"759385","Atlantis","1004",52.737590528958,3.88779468375724,"09/05/1800","00:10:00",7,188,1800-05-09 00:10:00 +"759386","Atlantis","1004",52.7510031365757,3.88584089571703,"09/05/1800"," 2:00:00",9,179,1800-05-09 02:00:00 +"759387","Atlantis","1004",52.7731042947639,3.91580103338696,"09/05/1800"," 3:34:00",7,182,1800-05-09 03:34:00 +"759388","Atlantis","1004",52.8288837654921,4.31737401986588,"09/05/1800"," 5:46:00",13,69,1800-05-09 05:46:00 +"759389","Atlantis","1004",52.9568596659838,4.76728857636266,"09/05/1800"," 7:44:00",0,201,1800-05-09 07:44:00 +"759390","Atlantis","1004",52.9570782294339,4.773146422741,"09/05/1800"," 9:20:00",0,201,1800-05-09 09:20:00 +"759391","Atlantis","1004",53.0433751153802,4.84980009037722,"09/05/1800","11:22:00",0,30,1800-05-09 11:22:00 +"759392","Atlantis","1004",53.0380895836716,4.85131246694084,"09/05/1800","13:44:00",0,30,1800-05-09 13:44:00 +"759393","Atlantis","1004",53.0465337839322,4.85769998701475,"09/05/1800","15:18:00",1,220,1800-05-09 15:18:00 +"759394","Atlantis","1004",53.0417720127171,4.84966114646476,"09/05/1800","17:26:00",0,220,1800-05-09 17:26:00 +"759395","Atlantis","1004",53.0473312837742,4.8458476259904,"09/05/1800","19:10:00",0,220,1800-05-09 19:10:00 +"1479869","Atlantis","1004",53.0392344388706,4.86036410922836,"05/09/1801","21:14:00",0,7,1801-09-05 21:14:00 +"1479870","Atlantis","1004",53.0423659209642,4.84364130577725,"05/09/1801","23:10:00",0,7,1801-09-05 23:10:00 +"1479871","Atlantis","1004",53.0457082244791,4.8539554268755,"06/09/1801"," 1:04:00",0,7,1801-09-06 01:04:00 +"1479872","Atlantis","1004",53.0434592928099,4.84513566746563,"06/09/1801"," 3:00:00",0,7,1801-09-06 03:00:00 +"1479873","Atlantis","1004",53.0451938302992,4.86182668678183,"06/09/1801"," 4:54:00",0,7,1801-09-06 04:54:00 +"1479874","Atlantis","1004",53.0445346946493,4.8507873154683,"06/09/1801"," 6:50:00",0,7,1801-09-06 06:50:00 +"1479875","Atlantis","1004",53.002573111725,4.703396786822,"06/09/1801"," 8:46:00",13,344,1801-09-06 08:46:00 +"1479876","Atlantis","1004",53.4089336687778,5.0119606510736,"06/09/1801","10:40:00",14,21,1801-09-06 10:40:00 +"1479877","Atlantis","1004",53.801261431986,5.27294915364497,"06/09/1801","12:36:00",13,19,1801-09-06 12:36:00 +"1479878","Atlantis","1004",54.1545395930749,5.46871859058831,"06/09/1801","14:30:00",0,146,1801-09-06 14:30:00 +"1479879","Atlantis","1004",54.3405738733252,5.50832696464471,"06/09/1801","16:26:00",2,352,1801-09-06 16:26:00 +"1479880","Atlantis","1004",54.4102298285351,5.82043598877732,"06/09/1801","18:22:00",7,66,1801-09-06 18:22:00 +"1479881","Atlantis","1004",54.426228788998,6.12755554681644,"06/09/1801","20:16:00",7,94,1801-09-06 20:16:00 +"1479882","Atlantis","1004",54.4099782569995,6.43957572595962,"06/09/1801","22:12:00",7,105,1801-09-06 22:12:00 +"1479883","Atlantis","1004",54.4076076412904,6.76971680895705,"07/09/1801","00: 6:00",7,83,1801-09-07 00:06:00 +"1479884","Atlantis","1004",54.3241370450603,7.07410840429273,"07/09/1801"," 2:02:00",8,139,1801-09-07 02:02:00 +"1479885","Atlantis","1004",54.3574650440938,7.05944719422143,"07/09/1801"," 3:58:00",7,339,1801-09-07 03:58:00 +"1479886","Atlantis","1004",54.4601323607746,6.82051229448803,"07/09/1801"," 5:52:00",8,67,1801-09-07 05:52:00 +"1479887","Atlantis","1004",54.4677537649819,6.81464160673134,"07/09/1801"," 7:48:00",6,280,1801-09-07 07:48:00 +"1479888","Atlantis","1004",54.4646518317168,6.42561809108965,"07/09/1801"," 9:42:00",4,50,1801-09-07 09:42:00 +"1479889","Atlantis","1004",54.6323866622695,6.28308853371627,"07/09/1801","11:38:00",4,347,1801-09-07 11:38:00 +"1479890","Atlantis","1004",54.7407147764154,6.20642765259836,"07/09/1801","13:34:00",7,271,1801-09-07 13:34:00 +"1479891","Atlantis","1004",54.4458198879589,6.34269926614314,"07/09/1801","17:24:00",7,175,1801-09-07 17:24:00 +"1484459","Atlantis","1004",54.3042274570023,6.28869009834714,"07/09/1801","19:18:00",7,171,1801-09-07 19:18:00 +"1484460","Atlantis","1004",54.2665358810923,6.42294426100794,"07/09/1801","21:14:00",7,104,1801-09-07 21:14:00 +"1484461","Atlantis","1004",54.3018953923653,6.37204595932365,"07/09/1801","23:10:00",7,340,1801-09-07 23:10:00 +"1484462","Atlantis","1004",54.4089804479578,6.33021399284154,"08/09/1801"," 1:04:00",7,160,1801-09-08 01:04:00 +"1484463","Atlantis","1004",54.2849679120462,6.36460551559832,"08/09/1801"," 3:00:00",7,143,1801-09-08 03:00:00 +"1484464","Atlantis","1004",54.4193463754496,6.28005679230485,"08/09/1801"," 4:56:00",1,169,1801-09-08 04:56:00 +"1484465","Atlantis","1004",54.469061608905,6.29027570619434,"08/09/1801"," 6:50:00",7,45,1801-09-08 06:50:00 +"1484466","Atlantis","1004",54.4791815830991,6.41702589099016,"08/09/1801"," 8:46:00",7,269,1801-09-08 08:46:00 +"1484467","Atlantis","1004",54.4944852960217,6.42948349237815,"08/09/1801","10:40:00",7,265,1801-09-08 10:40:00 +"1484468","Atlantis","1004",54.4989037375459,6.40030610878114,"08/09/1801","12:36:00",7,256,1801-09-08 12:36:00 +"1484469","Atlantis","1004",54.4632852229797,6.37143562685605,"08/09/1801","14:32:00",7,178,1801-09-08 14:32:00 +"1484470","Atlantis","1004",54.5497082880992,6.34017423263658,"08/09/1801","16:26:00",1,196,1801-09-08 16:26:00 +"1484471","Atlantis","1004",54.5108095879457,6.45603880997002,"08/09/1801","18:22:00",5,76,1801-09-08 18:22:00 +"1484472","Atlantis","1004",54.5897526759044,6.32924884794839,"08/09/1801","22:12:00",7,158,1801-09-08 22:12:00 +"1484473","Atlantis","1004",54.6439446739308,6.2994521883931,"09/09/1801","00: 6:00",7,340,1801-09-09 00:06:00 +"1484474","Atlantis","1004",54.5671804151614,6.35279353771917,"09/09/1801"," 3:58:00",7,316,1801-09-09 03:58:00 +"1484475","Atlantis","1004",54.5410447421949,6.37194776406698,"09/09/1801"," 5:52:00",7,321,1801-09-09 05:52:00 +"1484476","Atlantis","1004",54.5561973582567,6.35854177137557,"09/09/1801"," 7:48:00",6,144,1801-09-09 07:48:00 +"1484477","Atlantis","1004",54.5479606093392,6.35978876614291,"09/09/1801"," 9:44:00",5,129,1801-09-09 09:44:00 +"1484478","Atlantis","1004",54.5591612695865,6.32393997839186,"09/09/1801","11:38:00",7,125,1801-09-09 11:38:00 +"1484479","Atlantis","1004",54.5631873902399,6.31274870878365,"09/09/1801","13:34:00",7,127,1801-09-09 13:34:00 +"1484480","Atlantis","1004",54.5400646583047,5.88101539776568,"09/09/1801","19:20:00",8,210,1801-09-09 19:20:00 +"1484481","Atlantis","1004",54.3736211480629,5.70712033914495,"09/09/1801","21:14:00",7,205,1801-09-09 21:14:00 +"1484482","Atlantis","1004",54.1810130257835,5.55025073332246,"09/09/1801","23:10:00",1,287,1801-09-09 23:10:00 +"1484483","Atlantis","1004",54.050464284583,5.30522555922903,"10/09/1801"," 1:04:00",3,56,1801-09-10 01:04:00 +"1484484","Atlantis","1004",53.6680950363739,5.06148932672665,"10/09/1801"," 3:00:00",12,201,1801-09-10 03:00:00 +"1484485","Atlantis","1004",52.9694738769522,4.7607462142054,"10/09/1801"," 6:50:00",12,98,1801-09-10 06:50:00 +"1484486","Atlantis","1004",52.9548190539056,4.77466459030006,"10/09/1801"," 8:46:00",0,205,1801-09-10 08:46:00 +"1484487","Atlantis","1004",53.0374713666416,4.85216495376732,"10/09/1801","14:32:00",0,38,1801-09-10 14:32:00 +"1484488","Atlantis","1004",53.0447675585635,4.84586072135437,"10/09/1801","16:26:00",0,38,1801-09-10 16:26:00 +"1484526","Atlantis","1004",53.0413606886338,4.84757613166887,"14/09/1801"," 3:00:00",0,38,1801-09-14 03:00:00 +"1484527","Atlantis","1004",53.0450951574049,4.84715148185566,"14/09/1801"," 4:54:00",0,38,1801-09-14 04:54:00 +"1484528","Atlantis","1004",53.0406464351206,4.85143703424558,"14/09/1801"," 6:50:00",0,38,1801-09-14 06:50:00 +"1484529","Atlantis","1004",53.0430446803528,4.84530030579027,"14/09/1801"," 8:46:00",0,38,1801-09-14 08:46:00 +"1484530","Atlantis","1004",53.037455268668,4.85720497497078,"14/09/1801","12:36:00",0,38,1801-09-14 12:36:00 +"1484531","Atlantis","1004",53.023373687082,4.843852100241,"14/09/1801","14:30:00",13,225,1801-09-14 14:30:00 +"1484532","Atlantis","1004",53.3137383029661,4.84984935967438,"14/09/1801","16:26:00",14,29,1801-09-14 16:26:00 +"1489103","Atlantis","1004",53.684532109038,5.25305869249534,"14/09/1801","18:22:00",14,53,1801-09-14 18:22:00 +"1489104","Atlantis","1004",53.8541171006858,5.75524859377555,"14/09/1801","20:16:00",8,92,1801-09-14 20:16:00 +"1489105","Atlantis","1004",53.8501951747988,5.84582387269195,"14/09/1801","22:12:00",8,76,1801-09-14 22:12:00 +"1489106","Atlantis","1004",53.8670559113249,5.73439797494188,"15/09/1801","00: 6:00",2,84,1801-09-15 00:06:00 +"1489107","Atlantis","1004",53.8685538852597,5.77221434626263,"15/09/1801"," 2:02:00",7,260,1801-09-15 02:02:00 +"1489108","Atlantis","1004",53.8725501399008,5.83652434122376,"15/09/1801"," 3:58:00",7,272,1801-09-15 03:58:00 +"1489109","Atlantis","1004",53.8963125391765,5.94338676827121,"15/09/1801"," 5:52:00",7,66,1801-09-15 05:52:00 +"1489110","Atlantis","1004",53.879937529047,5.90622463855892,"15/09/1801"," 7:48:00",7,53,1801-09-15 07:48:00 +"1489111","Atlantis","1004",53.8693657158637,5.69412082180474,"15/09/1801"," 9:42:00",9,43,1801-09-15 09:42:00 +"1489112","Atlantis","1004",53.8584036023691,5.73459266796056,"15/09/1801","11:38:00",7,273,1801-09-15 11:38:00 +"1489113","Atlantis","1004",53.8682168937246,5.83647920089494,"15/09/1801","13:34:00",8,272,1801-09-15 13:34:00 +"1489114","Atlantis","1004",53.8650847992883,5.77495785683393,"15/09/1801","15:28:00",8,71,1801-09-15 15:28:00 +"1489115","Atlantis","1004",53.8627589943446,5.54314264584333,"15/09/1801","17:24:00",5,96,1801-09-15 17:24:00 +"1489116","Atlantis","1004",53.8699970897539,5.76539078393113,"15/09/1801","19:20:00",7,265,1801-09-15 19:20:00 +"1489117","Atlantis","1004",53.8652906453027,5.71726682690717,"15/09/1801","21:14:00",7,265,1801-09-15 21:14:00 +"1489118","Atlantis","1004",53.8797924722652,5.76718238819018,"15/09/1801","23:10:00",7,76,1801-09-15 23:10:00 +"1489119","Atlantis","1004",53.8728242186573,5.65034316006303,"16/09/1801"," 1:04:00",0,163,1801-09-16 01:04:00 +"1489120","Atlantis","1004",53.8768178287898,5.67966109550744,"16/09/1801"," 3:00:00",7,276,1801-09-16 03:00:00 +"1489121","Atlantis","1004",53.8767119087656,5.84437740060501,"16/09/1801"," 4:54:00",9,216,1801-09-16 04:54:00 +"1489122","Atlantis","1004",53.8571040251367,5.80541348170675,"16/09/1801"," 6:50:00",7,94,1801-09-16 06:50:00 +"1489123","Atlantis","1004",53.86117398979,5.6849948577825,"16/09/1801"," 8:46:00",1,111,1801-09-16 08:46:00 +"1489124","Atlantis","1004",53.8847196335993,5.75747202584986,"16/09/1801","10:40:00",7,277,1801-09-16 10:40:00 +"1489125","Atlantis","1004",53.8865175444987,5.82370227187965,"16/09/1801","12:36:00",7,297,1801-09-16 12:36:00 +"1489126","Atlantis","1004",53.8889822226651,5.8447759864293,"16/09/1801","14:30:00",7,132,1801-09-16 14:30:00 +"1489127","Atlantis","1004",53.880980653719,5.74309996145032,"16/09/1801","16:26:00",1,271,1801-09-16 16:26:00 +"1489128","Atlantis","1004",53.8731739558443,5.75912413315661,"16/09/1801","22:12:00",7,271,1801-09-16 22:12:00 +"1489129","Atlantis","1004",53.8546077947901,5.560710162987,"17/09/1801"," 2:02:00",3,308,1801-09-17 02:02:00 +"1489130","Atlantis","1004",53.7225079750507,5.51729022800457,"17/09/1801"," 3:58:00",13,194,1801-09-17 03:58:00 +"1489131","Atlantis","1004",53.4127458363031,5.04133749642409,"17/09/1801"," 5:52:00",13,232,1801-09-17 05:52:00 +"1489132","Atlantis","1004",53.102333352969,4.69778899929393,"17/09/1801"," 7:48:00",12,196,1801-09-17 07:48:00 +"1489133","Atlantis","1004",52.9545030079903,4.78054228164069,"17/09/1801"," 9:44:00",0,186,1801-09-17 09:44:00 +"1489134","Atlantis","1004",52.955386594255,4.78120440690126,"17/09/1801","11:38:00",0,186,1801-09-17 11:38:00 +"1489135","Atlantis","1004",52.9562979692249,4.78219287473056,"17/09/1801","13:34:00",0,186,1801-09-17 13:34:00 +"1489136","Atlantis","1004",53.0432727451702,4.85335730643943,"17/09/1801","15:28:00",5,288,1801-09-17 15:28:00 +"1489137","Atlantis","1004",53.0415816103467,4.84362992135249,"17/09/1801","17:24:00",0,266,1801-09-17 17:24:00 +"1489138","Atlantis","1004",53.0391540483581,4.84586369207036,"17/09/1801","19:20:00",0,266,1801-09-17 19:20:00 +"1489159","Atlantis","1004",53.041383995126,4.84326378515829,"19/09/1801","21:14:00",0,266,1801-09-19 21:14:00 +"1489160","Atlantis","1004",53.0392122051036,4.85009649136476,"19/09/1801","23:10:00",0,266,1801-09-19 23:10:00 +"1489161","Atlantis","1004",53.0415948440074,4.85166649456974,"20/09/1801"," 1:04:00",0,266,1801-09-20 01:04:00 +"1489162","Atlantis","1004",53.0358263468402,4.85625621794164,"20/09/1801"," 3:00:00",0,266,1801-09-20 03:00:00 +"1489163","Atlantis","1004",53.0370300040217,4.85940674014296,"20/09/1801"," 4:56:00",0,266,1801-09-20 04:56:00 +"1489164","Atlantis","1004",52.9802406552658,4.7601873349091,"20/09/1801"," 8:46:00",9,237,1801-09-20 08:46:00 +"1489165","Atlantis","1004",53.353141011151,4.91828634655382,"20/09/1801","10:40:00",14,39,1801-09-20 10:40:00 +"1489166","Atlantis","1004",53.7139946421618,5.37867092360742,"20/09/1801","12:36:00",14,39,1801-09-20 12:36:00 +"1489167","Atlantis","1004",53.8703608419439,5.76477424832154,"20/09/1801","14:32:00",7,92,1801-09-20 14:32:00 +"1489168","Atlantis","1004",53.9073997766292,5.72150444294047,"20/09/1801","16:26:00",7,256,1801-09-20 16:26:00 +"1489169","Atlantis","1004",53.904886003525,5.57220428710524,"20/09/1801","18:22:00",7,99,1801-09-20 18:22:00 +"1489170","Atlantis","1004",53.8877062355555,5.48213320673164,"20/09/1801","20:18:00",5,80,1801-09-20 20:18:00 +"1489171","Atlantis","1004",53.8608302167985,5.40583233108465,"20/09/1801","22:12:00",6,279,1801-09-20 22:12:00 +"1489172","Atlantis","1004",53.8745709653744,5.52569020885136,"21/09/1801","00: 8:00",8,231,1801-09-21 00:08:00 +"1489173","Atlantis","1004",53.8770671329903,5.53651248163078,"21/09/1801"," 2:02:00",7,79,1801-09-21 02:02:00 +"1489174","Atlantis","1004",53.8674251821097,5.44649115431681,"21/09/1801"," 3:58:00",4,59,1801-09-21 03:58:00 +"1489175","Atlantis","1004",53.8890498611685,5.55427154102828,"21/09/1801"," 5:52:00",4,269,1801-09-21 05:52:00 +"1489176","Atlantis","1004",53.9126509298701,5.36414184421208,"21/09/1801"," 7:48:00",4,300,1801-09-21 07:48:00 +"1493756","Atlantis","1004",53.9220546782175,5.19734405987803,"21/09/1801"," 9:44:00",3,256,1801-09-21 09:44:00 +"1493757","Atlantis","1004",53.9030618058606,5.21180047482625,"21/09/1801","11:38:00",4,97,1801-09-21 11:38:00 +"1493758","Atlantis","1004",53.8746700852816,5.48578213242814,"21/09/1801","13:34:00",6,281,1801-09-21 13:34:00 +"1493759","Atlantis","1004",53.8945356517788,5.47507692701649,"21/09/1801","17:24:00",3,104,1801-09-21 17:24:00 +"1493760","Atlantis","1004",53.8677338226954,5.54203094118833,"21/09/1801","19:20:00",9,258,1801-09-21 19:20:00 +"1493761","Atlantis","1004",53.7728471342791,5.04146906879358,"21/09/1801","21:14:00",9,244,1801-09-21 21:14:00 +"1493762","Atlantis","1004",53.6680449370253,4.7111601828821,"21/09/1801","23:10:00",12,168,1801-09-21 23:10:00 +"1493763","Atlantis","1004",53.3509487155816,4.87574424170423,"22/09/1801"," 1:04:00",9,269,1801-09-22 01:04:00 +"1493764","Atlantis","1004",53.1749855099181,4.66625632005092,"22/09/1801"," 3:00:00",12,169,1801-09-22 03:00:00 +"1493765","Atlantis","1004",52.9559768810533,4.77402996963076,"22/09/1801"," 4:56:00",0,160,1801-09-22 04:56:00 +"1493766","Atlantis","1004",52.9582675949638,4.78059387337603,"22/09/1801"," 6:50:00",5,334,1801-09-22 06:50:00 +"1493767","Atlantis","1004",53.0405508777346,4.8471540538827,"22/09/1801"," 8:46:00",5,284,1801-09-22 08:46:00 +"1493768","Atlantis","1004",53.0363904897068,4.85920494180359,"22/09/1801","10:40:00",0,15,1801-09-22 10:40:00 +"1493769","Atlantis","1004",53.0450095155849,4.86082073102891,"22/09/1801","12:36:00",0,15,1801-09-22 12:36:00 +"1493770","Atlantis","1004",53.038896302205,4.8459733800143,"22/09/1801","14:32:00",0,15,1801-09-22 14:32:00 +"1493771","Atlantis","1004",53.0413674629563,4.84377708098665,"22/09/1801","16:26:00",0,15,1801-09-22 16:26:00 +"1493819","Atlantis","1004",53.0424839479635,4.85790369552746,"26/09/1801","20:16:00",0,269,1801-09-26 20:16:00 +"1493820","Atlantis","1004",53.0458899974492,4.84896117741428,"26/09/1801","22:12:00",0,269,1801-09-26 22:12:00 +"1493821","Atlantis","1004",53.0386887827418,4.85392880304903,"27/09/1801","00: 6:00",0,269,1801-09-27 00:06:00 +"1493822","Atlantis","1004",53.0459254124733,4.86072373101208,"27/09/1801"," 2:02:00",0,269,1801-09-27 02:02:00 +"1493823","Atlantis","1004",53.0456877512224,4.84478079823498,"27/09/1801"," 3:58:00",0,269,1801-09-27 03:58:00 +"1493824","Atlantis","1004",53.0435075528123,4.85599908633809,"27/09/1801"," 5:52:00",0,269,1801-09-27 05:52:00 +"1493825","Atlantis","1004",53.0393261683029,4.85116349788103,"27/09/1801"," 7:48:00",0,269,1801-09-27 07:48:00 +"1493826","Atlantis","1004",53.1771236844067,4.74853564695827,"27/09/1801"," 9:42:00",12,13,1801-09-27 09:42:00 +"1493827","Atlantis","1004",53.516384563677,5.00236942001246,"27/09/1801","11:38:00",13,39,1801-09-27 11:38:00 +"1493828","Atlantis","1004",53.8479856063551,5.10022544643842,"27/09/1801","13:34:00",1,235,1801-09-27 13:34:00 +"1493829","Atlantis","1004",53.885575038251,5.42934975891002,"27/09/1801","15:28:00",3,94,1801-09-27 15:28:00 +"1498390","Atlantis","1004",53.8732316767797,5.40910662597418,"27/09/1801","17:24:00",5,282,1801-09-27 17:24:00 +"1498391","Atlantis","1004",53.9012167348964,5.7450635034591,"27/09/1801","19:18:00",7,90,1801-09-27 19:18:00 +"1498392","Atlantis","1004",53.9826825512056,5.55440173270274,"27/09/1801","21:14:00",7,342,1801-09-27 21:14:00 +"1498393","Atlantis","1004",54.1746558995764,5.51592316549458,"27/09/1801","23:10:00",7,358,1801-09-27 23:10:00 +"1498394","Atlantis","1004",54.3429850017154,5.5216509736022,"28/09/1801"," 1:04:00",8,270,1801-09-28 01:04:00 +"1498395","Atlantis","1004",54.5438841840178,5.43429128994607,"28/09/1801"," 3:00:00",7,331,1801-09-28 03:00:00 +"1498396","Atlantis","1004",54.6265176144666,5.17080919660069,"28/09/1801"," 4:56:00",5,253,1801-09-28 04:56:00 +"1498397","Atlantis","1004",54.6130036303354,5.14441204735357,"28/09/1801"," 6:50:00",7,92,1801-09-28 06:50:00 +"1498398","Atlantis","1004",54.6442671393706,5.07349199787155,"28/09/1801"," 8:46:00",7,96,1801-09-28 08:46:00 +"1498399","Atlantis","1004",54.6556689588758,5.08711787709314,"28/09/1801","10:40:00",7,269,1801-09-28 10:40:00 +"1498400","Atlantis","1004",54.6749579561036,5.23317885142658,"28/09/1801","12:36:00",7,60,1801-09-28 12:36:00 +"1498401","Atlantis","1004",54.6885401250087,5.37116418926045,"28/09/1801","14:32:00",7,63,1801-09-28 14:32:00 +"1498402","Atlantis","1004",54.6870078116669,5.27801883398648,"28/09/1801","16:26:00",7,100,1801-09-28 16:26:00 +"1498403","Atlantis","1004",54.6775877113487,5.15577065772656,"28/09/1801","18:22:00",5,80,1801-09-28 18:22:00 +"1498404","Atlantis","1004",54.722408869857,5.09585706325993,"28/09/1801","20:16:00",8,263,1801-09-28 20:16:00 +"1498405","Atlantis","1004",54.554771568595,5.15765452293772,"28/09/1801","22:12:00",7,179,1801-09-28 22:12:00 +"1498406","Atlantis","1004",54.5472746419385,4.9549135602545,"29/09/1801"," 2:02:00",7,260,1801-09-29 02:02:00 +"1498407","Atlantis","1004",54.4777533122655,4.833164861314,"29/09/1801"," 3:58:00",7,224,1801-09-29 03:58:00 +"1498408","Atlantis","1004",54.5357829391104,4.99646874918137,"29/09/1801"," 5:52:00",7,75,1801-09-29 05:52:00 +"1498409","Atlantis","1004",54.6298046897952,5.30347497604322,"29/09/1801"," 7:48:00",8,54,1801-09-29 07:48:00 +"1498410","Atlantis","1004",54.7311634111167,5.55805744402949,"29/09/1801"," 9:44:00",7,79,1801-09-29 09:44:00 +"1498411","Atlantis","1004",54.7383905305443,5.57868081925437,"29/09/1801"," 9:46:00",7,79,1801-09-29 09:46:00 +"1498412","Atlantis","1004",54.860246430621,5.72348624049779,"29/09/1801","11:38:00",5,154,1801-09-29 11:38:00 +"1498413","Atlantis","1004",54.7078279539039,5.95806460902467,"29/09/1801","13:34:00",7,94,1801-09-29 13:34:00 +"1498414","Atlantis","1004",54.7347290559299,5.91020815948211,"29/09/1801","15:28:00",7,85,1801-09-29 15:28:00 +"1498415","Atlantis","1004",54.7171370475665,5.91121322342381,"29/09/1801","17:24:00",8,265,1801-09-29 17:24:00 +"1498416","Atlantis","1004",54.7189302717722,5.90433417737949,"29/09/1801","19:18:00",7,5,1801-09-29 19:18:00 +"1498417","Atlantis","1004",54.8243315741261,5.76416948076803,"29/09/1801","21:14:00",4,156,1801-09-29 21:14:00 +"1498418","Atlantis","1004",54.7710343252285,6.09365564829856,"29/09/1801","23:10:00",7,113,1801-09-29 23:10:00 +"1498419","Atlantis","1004",54.8287549877102,5.99204119027685,"30/09/1801"," 1:04:00",7,25,1801-09-30 01:04:00 +"1498420","Atlantis","1004",54.8456141366963,5.99877673429437,"30/09/1801"," 3:00:00",7,11,1801-09-30 03:00:00 +"1498421","Atlantis","1004",54.7943309591771,5.88489189432747,"30/09/1801"," 4:54:00",7,335,1801-09-30 04:54:00 +"1498422","Atlantis","1004",54.8511111874399,5.65373958104662,"30/09/1801"," 6:50:00",7,207,1801-09-30 06:50:00 +"1498423","Atlantis","1004",54.6627870871769,5.64673999824654,"30/09/1801"," 8:46:00",7,203,1801-09-30 08:46:00 +"1498424","Atlantis","1004",54.4420913820355,5.59335672736168,"30/09/1801","10:40:00",7,174,1801-09-30 10:40:00 +"1498425","Atlantis","1004",54.3057493562344,5.12166182311997,"30/09/1801","14:32:00",7,306,1801-09-30 14:32:00 +"1498426","Atlantis","1004",54.3742666170541,5.04340487123001,"30/09/1801","16:26:00",7,183,1801-09-30 16:26:00 +"1498427","Atlantis","1004",54.3740860519977,4.98883475052286,"30/09/1801","18:22:00",8,196,1801-09-30 18:22:00 +"1498428","Atlantis","1004",54.180676986238,4.9536893641809,"30/09/1801","20:16:00",7,188,1801-09-30 20:16:00 +"1498429","Atlantis","1004",53.9533367969156,4.83463261436485,"30/09/1801","22:12:00",7,199,1801-09-30 22:12:00 +"1498430","Atlantis","1004",53.7830785151166,4.69759559225664,"01/10/1801","00: 8:00",7,211,1801-10-01 00:08:00 +"1498431","Atlantis","1004",53.5950123654883,4.60024732373096,"01/10/1801"," 2:02:00",7,204,1801-10-01 02:02:00 +"1498432","Atlantis","1004",53.3870545951696,4.49037663356774,"01/10/1801"," 3:58:00",2,275,1801-10-01 03:58:00 +"1498433","Atlantis","1004",53.1399426974538,4.65807567935437,"01/10/1801"," 5:52:00",13,172,1801-10-01 05:52:00 +"1498434","Atlantis","1004",52.956249852933,4.78435198619682,"01/10/1801"," 7:48:00",0,174,1801-10-01 07:48:00 +"1498435","Atlantis","1004",52.9526760224863,4.76897201792058,"01/10/1801"," 9:42:00",0,174,1801-10-01 09:42:00 +"1498436","Atlantis","1004",53.0423439018903,4.84582812025957,"01/10/1801","13:34:00",0,286,1801-10-01 13:34:00 +"1498437","Atlantis","1004",53.0374628454484,4.85809543730877,"01/10/1801","15:28:00",0,14,1801-10-01 15:28:00 +"1498438","Atlantis","1004",53.0369052168787,4.86236234583612,"01/10/1801","17:24:00",0,14,1801-10-01 17:24:00 +"1498439","Atlantis","1004",53.0389626972415,4.86214114801679,"01/10/1801","19:20:00",0,14,1801-10-01 19:20:00 +"764928","Atlantis","1013",53.0445871164692,4.86463103540148,"01/06/1800","20:54:00",0,7,1800-06-01 20:54:00 +"764929","Atlantis","1013",53.0460339151365,4.86373789428174,"01/06/1800","21:32:00",0,7,1800-06-01 21:32:00 +"764930","Atlantis","1013",53.045582758402,4.85218078402244,"01/06/1800","21:52:00",0,7,1800-06-01 21:52:00 +"764931","Atlantis","1013",53.0386538732117,4.85375051884819,"01/06/1800","22:54:00",0,7,1800-06-01 22:54:00 +"764932","Atlantis","1013",53.0454681639113,4.85509372117836,"01/06/1800","23:28:00",0,7,1800-06-01 23:28:00 +"764933","Atlantis","1013",53.0464968574252,4.85186548210587,"01/06/1800","23:52:00",0,7,1800-06-01 23:52:00 +"764934","Atlantis","1013",53.0436705400185,4.84986880359613,"02/06/1800","00:54:00",0,7,1800-06-02 00:54:00 +"713052","Atlantis","1013",53.0469179774001,4.86592301778868,"02/06/1800"," 1:22:00",0,7,1800-06-02 01:22:00 +"713053","Atlantis","1013",53.0474315711665,4.84872884988226,"02/06/1800"," 1:54:00",0,7,1800-06-02 01:54:00 +"713054","Atlantis","1013",53.0385141079077,4.85337242881674,"02/06/1800"," 2:54:00",0,7,1800-06-02 02:54:00 +"713055","Atlantis","1013",53.0457420870517,4.85971545580216,"02/06/1800"," 3:18:00",0,7,1800-06-02 03:18:00 +"713056","Atlantis","1013",53.0467639279803,4.85813313893415,"02/06/1800"," 3:54:00",0,7,1800-06-02 03:54:00 +"713057","Atlantis","1013",53.0469054077631,4.85113428926002,"02/06/1800"," 4:54:00",0,7,1800-06-02 04:54:00 +"713058","Atlantis","1013",53.0453492441652,4.86283408027515,"02/06/1800"," 5:14:00",0,7,1800-06-02 05:14:00 +"713059","Atlantis","1013",53.047797080548,4.84913006685581,"02/06/1800"," 5:54:00",0,7,1800-06-02 05:54:00 +"713060","Atlantis","1013",53.0466775328568,4.84897106715851,"02/06/1800"," 6:54:00",0,7,1800-06-02 06:54:00 +"713061","Atlantis","1013",53.0401929945913,4.8545125005804,"02/06/1800"," 7:08:00",0,7,1800-06-02 07:08:00 +"713062","Atlantis","1013",53.0403545004032,4.86420177006069,"02/06/1800"," 7:54:00",0,7,1800-06-02 07:54:00 +"713063","Atlantis","1013",53.0034476027573,4.80379352284968,"02/06/1800"," 8:54:00",10,222,1800-06-02 08:54:00 +"713064","Atlantis","1013",52.9855472481167,4.77436282677576,"02/06/1800"," 9:04:00",9,233,1800-06-02 09:04:00 +"713065","Atlantis","1013",53.0727167975032,4.66835210822243,"02/06/1800"," 9:54:00",11,327,1800-06-02 09:54:00 +"713066","Atlantis","1013",53.2178393587619,4.48628928215336,"02/06/1800","10:54:00",11,324,1800-06-02 10:54:00 +"713067","Atlantis","1013",53.3641471679481,4.31758947358839,"02/06/1800","11:54:00",11,330,1800-06-02 11:54:00 +"713068","Atlantis","1013",53.500557712812,4.16456363636069,"02/06/1800","12:54:00",10,335,1800-06-02 12:54:00 +"713069","Atlantis","1013",53.5113623653916,4.15619658119418,"02/06/1800","12:56:00",10,331,1800-06-02 12:56:00 +"713070","Atlantis","1013",53.6418743312932,4.00462705611996,"02/06/1800","13:54:00",7,31,1800-06-02 13:54:00 +"713071","Atlantis","1013",53.6839725131998,4.08100425949414,"02/06/1800","14:50:00",4,53,1800-06-02 14:50:00 +"713072","Atlantis","1013",53.6884169258093,4.08944364615716,"02/06/1800","14:54:00",4,59,1800-06-02 14:54:00 +"713073","Atlantis","1013",53.7246389868674,4.18202278881054,"02/06/1800","15:54:00",4,57,1800-06-02 15:54:00 +"713074","Atlantis","1013",53.7574875975703,4.25733833231777,"02/06/1800","16:44:00",4,56,1800-06-02 16:44:00 +"713075","Atlantis","1013",53.7668795018671,4.27778346516564,"02/06/1800","16:54:00",4,55,1800-06-02 16:54:00 +"713076","Atlantis","1013",53.8127548598545,4.33073110030033,"02/06/1800","17:54:00",2,26,1800-06-02 17:54:00 +"713077","Atlantis","1013",53.8369536469337,4.39240467538312,"02/06/1800","18:40:00",4,48,1800-06-02 18:40:00 +"713078","Atlantis","1013",53.8510140035497,4.41071273786668,"02/06/1800","18:54:00",4,50,1800-06-02 18:54:00 +"713079","Atlantis","1013",53.8924040693426,4.47646900190599,"02/06/1800","19:54:00",4,11,1800-06-02 19:54:00 +"713080","Atlantis","1013",53.9384862843608,4.49087864725944,"02/06/1800","20:36:00",4,345,1800-06-02 20:36:00 +"713081","Atlantis","1013",54.0248097582464,4.47662326706853,"02/06/1800","21:54:00",4,359,1800-06-02 21:54:00 +"713082","Atlantis","1013",54.0539242369477,4.47393683302123,"02/06/1800","22:30:00",2,48,1800-06-02 22:30:00 +"713083","Atlantis","1013",54.0593657210646,4.46216298739892,"02/06/1800","22:54:00",4,271,1800-06-02 22:54:00 +"713084","Atlantis","1013",54.0724557867432,4.35473273146153,"02/06/1800","23:54:00",4,295,1800-06-02 23:54:00 +"713085","Atlantis","1013",54.0937221106337,4.31170431319065,"03/06/1800","00:26:00",4,298,1800-06-03 00:26:00 +"713086","Atlantis","1013",54.1054944752567,4.25656835188344,"03/06/1800","00:54:00",4,325,1800-06-03 00:54:00 +"713087","Atlantis","1013",54.1705195882171,4.24435283387452,"03/06/1800"," 1:54:00",4,0,1800-06-03 01:54:00 +"713088","Atlantis","1013",54.1975776973614,4.22664754016046,"03/06/1800"," 2:20:00",4,357,1800-06-03 02:20:00 +"713089","Atlantis","1013",54.2366604352007,4.23313521215413,"03/06/1800"," 2:54:00",5,65,1800-06-03 02:54:00 +"713090","Atlantis","1013",54.2204566313564,4.29476330519002,"03/06/1800"," 3:54:00",5,126,1800-06-03 03:54:00 +"713091","Atlantis","1013",54.1669422668009,4.36375270065013,"03/06/1800"," 4:54:00",4,174,1800-06-03 04:54:00 +"713092","Atlantis","1013",54.0995090839206,4.36887693024613,"03/06/1800"," 5:54:00",4,170,1800-06-03 05:54:00 +"713093","Atlantis","1013",54.0812382086078,4.38327260411251,"03/06/1800"," 6:10:00",4,175,1800-06-03 06:10:00 +"713094","Atlantis","1013",54.0545888009551,4.42777723629028,"03/06/1800"," 6:54:00",4,97,1800-06-03 06:54:00 +"713095","Atlantis","1013",54.1262567106523,4.46087200947013,"03/06/1800"," 8:06:00",3,1,1800-06-03 08:06:00 +"713096","Atlantis","1013",54.0963784918478,4.46367096793186,"03/06/1800"," 8:54:00",4,181,1800-06-03 08:54:00 +"713097","Atlantis","1013",54.0613508963357,4.42303086384758,"03/06/1800"," 9:54:00",4,278,1800-06-03 09:54:00 +"713098","Atlantis","1013",54.0543557870337,4.4176028245287,"03/06/1800","10:02:00",0,0,1800-06-03 10:02:00 +"713099","Atlantis","1013",54.1073659311994,4.37388325918093,"03/06/1800","10:54:00",4,346,1800-06-03 10:54:00 +"713100","Atlantis","1013",54.1567246059137,4.42744875764754,"03/06/1800","11:54:00",4,74,1800-06-03 11:54:00 +"713101","Atlantis","1013",54.1540674390523,4.44945978038013,"03/06/1800","11:56:00",4,72,1800-06-03 11:56:00 +"713102","Atlantis","1013",54.1309240520871,4.48307388088107,"03/06/1800","12:54:00",5,203,1800-06-03 12:54:00 +"713103","Atlantis","1013",54.0610545323333,4.4713629917521,"03/06/1800","13:52:00",4,180,1800-06-03 13:52:00 +"713104","Atlantis","1013",54.0594972396912,4.4586311679706,"03/06/1800","13:54:00",4,181,1800-06-03 13:54:00 +"713105","Atlantis","1013",53.9904294324801,4.46561224362068,"03/06/1800","14:54:00",4,175,1800-06-03 14:54:00 +"713106","Atlantis","1013",53.9322131360965,4.49262967333104,"03/06/1800","15:46:00",4,161,1800-06-03 15:46:00 +"713107","Atlantis","1013",53.9318582300474,4.48370686313696,"03/06/1800","15:56:00",4,175,1800-06-03 15:56:00 +"713108","Atlantis","1013",53.8810512356898,4.46390603309311,"03/06/1800","16:56:00",0,291,1800-06-03 16:56:00 +"713109","Atlantis","1013",53.9331506388229,4.48167346574925,"03/06/1800","17:42:00",4,351,1800-06-03 17:42:00 +"713110","Atlantis","1013",53.9497527976078,4.488143903994,"03/06/1800","17:54:00",4,341,1800-06-03 17:54:00 +"713111","Atlantis","1013",54.0151343890261,4.46992781254835,"03/06/1800","18:56:00",4,355,1800-06-03 18:56:00 +"713112","Atlantis","1013",54.0610182500887,4.45706815000065,"03/06/1800","19:38:00",4,4,1800-06-03 19:38:00 +"713113","Atlantis","1013",54.0798506334983,4.46390224244911,"03/06/1800","19:54:00",4,7,1800-06-03 19:54:00 +"713114","Atlantis","1013",54.1449411457027,4.46169580347464,"03/06/1800","20:56:00",4,339,1800-06-03 20:56:00 +"713115","Atlantis","1013",54.1783779362664,4.47027558408677,"03/06/1800","21:32:00",4,4,1800-06-03 21:32:00 +"713116","Atlantis","1013",54.2072464842112,4.47279666218813,"03/06/1800","21:56:00",4,359,1800-06-03 21:56:00 +"713117","Atlantis","1013",54.2684494530074,4.45861996958871,"03/06/1800","22:56:00",4,5,1800-06-03 22:56:00 +"713118","Atlantis","1013",54.3044780273423,4.4714724832084,"03/06/1800","23:28:00",4,5,1800-06-03 23:28:00 +"713119","Atlantis","1013",54.3290568605065,4.49945971040707,"03/06/1800","23:56:00",4,73,1800-06-03 23:56:00 +"713120","Atlantis","1013",54.3092585720257,4.61139908613078,"04/06/1800","00:56:00",4,127,1800-06-04 00:56:00 +"713121","Atlantis","1013",54.2854070612853,4.6452328699939,"04/06/1800"," 1:26:00",4,124,1800-06-04 01:26:00 +"713122","Atlantis","1013",54.3234183424842,4.57637164594792,"04/06/1800"," 2:56:00",4,306,1800-06-04 02:56:00 +"713123","Atlantis","1013",54.3343880567062,4.5462838120209,"04/06/1800"," 3:18:00",4,265,1800-06-04 03:18:00 +"713124","Atlantis","1013",54.3190655451552,4.48215522045642,"04/06/1800"," 3:56:00",4,198,1800-06-04 03:56:00 +"713125","Atlantis","1013",54.2534568882696,4.4595540603064,"04/06/1800"," 4:56:00",4,178,1800-06-04 04:56:00 +"764935","Atlantis","1013",54.2255252670785,4.45997055232898,"04/06/1800"," 5:14:00",4,181,1800-06-04 05:14:00 +"764936","Atlantis","1013",54.1827902029455,4.4608511306718,"04/06/1800"," 5:56:00",4,178,1800-06-04 05:56:00 +"764937","Atlantis","1013",54.1706070162682,4.46398532980122,"04/06/1800"," 6:56:00",4,10,1800-06-04 06:56:00 +"764938","Atlantis","1013",54.1866767612239,4.45307082518656,"04/06/1800"," 7:08:00",4,9,1800-06-04 07:08:00 +"764939","Atlantis","1013",54.2280650737668,4.49196241352055,"04/06/1800"," 7:56:00",4,190,1800-06-04 07:56:00 +"764940","Atlantis","1013",54.168189148712,4.45871613687836,"04/06/1800"," 8:56:00",4,181,1800-06-04 08:56:00 +"764941","Atlantis","1013",54.1581233380227,4.46346532505937,"04/06/1800"," 9:04:00",4,183,1800-06-04 09:04:00 +"764942","Atlantis","1013",54.0991315294104,4.46677847264335,"04/06/1800"," 9:56:00",4,184,1800-06-04 09:56:00 +"764943","Atlantis","1013",54.059838255534,4.46675562749244,"04/06/1800","10:56:00",0,184,1800-06-04 10:56:00 +"764944","Atlantis","1013",54.0678556052963,4.45623684887309,"04/06/1800","10:58:00",5,180,1800-06-04 10:58:00 +"764945","Atlantis","1013",54.0021821238678,4.46337098925188,"04/06/1800","11:56:00",4,179,1800-06-04 11:56:00 +"764946","Atlantis","1013",54.0226125252964,4.47464818236604,"04/06/1800","12:54:00",4,346,1800-06-04 12:54:00 +"764947","Atlantis","1013",54.0220069744927,4.46630089172907,"04/06/1800","12:56:00",4,346,1800-06-04 12:56:00 +"764948","Atlantis","1013",54.0765229104748,4.49666681366227,"04/06/1800","13:56:00",4,52,1800-06-04 13:56:00 +"764949","Atlantis","1013",54.1362038454637,4.50310239016358,"04/06/1800","14:56:00",2,357,1800-06-04 14:56:00 +"764950","Atlantis","1013",54.1131505484656,4.46757924990356,"04/06/1800","15:56:00",4,194,1800-06-04 15:56:00 +"764951","Atlantis","1013",54.0516472674878,4.46887763362378,"04/06/1800","16:44:00",4,179,1800-06-04 16:44:00 +"764952","Atlantis","1013",54.0385747598121,4.4678044912871,"04/06/1800","16:56:00",4,181,1800-06-04 16:56:00 +"764953","Atlantis","1013",53.9971018870021,4.48401023243088,"04/06/1800","17:56:00",0,182,1800-06-04 17:56:00 +"764954","Atlantis","1013",54.0431060008463,4.46214861033019,"04/06/1800","18:40:00",4,360,1800-06-04 18:40:00 +"764955","Atlantis","1013",54.0601452184292,4.45549645537045,"04/06/1800","18:56:00",4,359,1800-06-04 18:56:00 +"764956","Atlantis","1013",54.0924504707051,4.47381234739814,"04/06/1800","19:56:00",4,6,1800-06-04 19:56:00 +"764957","Atlantis","1013",54.1416818769146,4.46091814263724,"04/06/1800","20:34:00",4,2,1800-06-04 20:34:00 +"764958","Atlantis","1013",54.1626328017148,4.45924129400402,"04/06/1800","20:56:00",4,356,1800-06-04 20:56:00 +"764959","Atlantis","1013",54.2267736967779,4.45630392862018,"04/06/1800","21:56:00",4,1,1800-06-04 21:56:00 +"764960","Atlantis","1013",54.2712123377393,4.4624219927555,"04/06/1800","22:30:00",4,9,1800-06-04 22:30:00 +"764961","Atlantis","1013",54.3002301167794,4.48569389009755,"04/06/1800","22:56:00",4,8,1800-06-04 22:56:00 +"764962","Atlantis","1013",54.3280116153839,4.55928898821492,"04/06/1800","23:56:00",4,121,1800-06-04 23:56:00 +"764963","Atlantis","1013",54.3144435860305,4.60334784745145,"05/06/1800","00:26:00",3,124,1800-06-05 00:26:00 +"764964","Atlantis","1013",54.302965775717,4.62922692467924,"05/06/1800","00:56:00",4,308,1800-06-05 00:56:00 +"764965","Atlantis","1013",54.3263973832401,4.53139137225505,"05/06/1800"," 1:56:00",4,261,1800-06-05 01:56:00 +"764966","Atlantis","1013",54.3257364773415,4.48515264021419,"05/06/1800"," 2:20:00",4,209,1800-06-05 02:20:00 +"764967","Atlantis","1013",54.2888540545716,4.46404573750775,"05/06/1800"," 2:56:00",4,194,1800-06-05 02:56:00