Fabio Sinibaldi 2018-07-24 16:16:33 +00:00
parent 7dcddb56ed
commit ea7c7a184a
8 changed files with 113 additions and 93 deletions

View File

@ -1,7 +1,7 @@
package org.gcube.accounting.accounting.summary.access;
import java.sql.SQLException;
import java.util.Date;
import java.time.Instant;
import java.util.Set;
import org.gcube.accounting.accounting.summary.access.impl.AccountingDaoImpl;
@ -27,7 +27,7 @@ public interface AccountingDao {
public ScopeDescriptor getTree(Object request)throws ParameterException, Exception;
public Report getReportByScope(ScopeDescriptor desc, Date from, Date to, MeasureResolution resolution) throws ParameterException, SQLException;
public Report getReportByScope(ScopeDescriptor desc, Instant from, Instant to, MeasureResolution resolution) throws ParameterException, SQLException;
public Set<Dimension> getDimensions() throws SQLException;

View File

@ -4,11 +4,14 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.Period;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;
@ -30,12 +33,12 @@ import org.gcube.accounting.accounting.summary.access.model.update.UpdateReport;
import lombok.extern.slf4j.Slf4j;
import static java.time.temporal.TemporalAdjusters.*;
@Slf4j
public class AccountingDaoImpl implements AccountingDao{
private static final ZoneId UTC=ZoneId.of("UTC");
private ContextTreeProvider treeProvider=null;
private ConnectionManager connectionManager=null;
@ -57,21 +60,19 @@ public class AccountingDaoImpl implements AccountingDao{
@Override
public Report getReportByScope(ScopeDescriptor desc, Date from, Date to, MeasureResolution resolution) throws SQLException, ParameterException {
public Report getReportByScope(ScopeDescriptor desc, Instant from, Instant to, MeasureResolution resolution) throws SQLException, ParameterException {
DateTimeFormatter formatter=getFormatter(resolution);
LocalDateTime fromDate=from.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
LocalDateTime toDate=to.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
log.info("Loading report {} for {} between {} and {} ",resolution,desc.getId(),formatter.format(fromDate),formatter.format(toDate));
log.info("Loading report {} for {} between {} and {} ",resolution,desc.getId(),
formatter.format(from.atZone(UTC).toLocalDateTime()),formatter.format(to.atZone(UTC).toLocalDateTime()));
long startReportTime=System.currentTimeMillis();
fromDate=fixLowerBound(fromDate, resolution);
toDate=fixUpperBound(toDate, resolution);
if(fromDate.isAfter(toDate)) throw new ParameterException("Irregular time interval: \"From\" parameter cannot be after \"To\" parameter.");
int timeSlices=getRangeSize(fromDate, toDate, resolution);
if(from.isAfter(to)) throw new ParameterException("Irregular time interval: \"From\" parameter cannot be after \"To\" parameter.");
int timeSlices=(int)getRangeSize(from, to, resolution);
// if(timeSlices==0) {
// log.debug("Time slices is {}. Going to increment \"To\" parameter.",timeSlices);
// toDate=increment(toDate,resolution,1);
@ -86,7 +87,7 @@ public class AccountingDaoImpl implements AccountingDao{
//load available dimensions in time slice
ResultSet dimensionRS=queries.getAvailableDimensions(fromDate, toDate, desc, resolution);
ResultSet dimensionRS=queries.getAvailableDimensions(from, to, desc, resolution);
LinkedList<Dimension> foundDimensions=new LinkedList<>();
while(dimensionRS.next()){
String id=dimensionRS.getString(DIMENSIONS.ID);
@ -107,13 +108,13 @@ public class AccountingDaoImpl implements AccountingDao{
// Report 1 series for selected Scope
reports.add(new ReportElement(desc.getName()+" "+yLabel,category,
xLabel,yLabel,new Series[]{getSeries(queries, fromDate, toDate, entry, desc, resolution, timeSlices)}));
xLabel,yLabel,new Series[]{getSeries(queries, from, to, entry, desc, resolution, timeSlices)}));
// Report 2 series for each children
if(desc.hasChildren()) {
LinkedList<Series> childrenSeries=new LinkedList<>();
for(ScopeDescriptor child:desc.getChildren()){
childrenSeries.add(getSeries(queries, fromDate, toDate, entry, child, resolution, timeSlices));
childrenSeries.add(getSeries(queries, from, to, entry, child, resolution, timeSlices));
}
reports.add(new ReportElement(desc.getName()+" children "+yLabel,category,
xLabel,yLabel,childrenSeries.toArray(new Series[childrenSeries.size()])));
@ -137,19 +138,27 @@ public class AccountingDaoImpl implements AccountingDao{
}
private LocalDateTime fixLowerBound(LocalDateTime toFix,MeasureResolution resolution) throws ParameterException {
switch(resolution) {
case MONTHLY : return toFix.with(firstDayOfMonth()).withHour(0).withMinute(0).withSecond(0).withNano(0);
default : throw new ParameterException("Invalid resolution "+resolution);
}
}
private LocalDateTime fixUpperBound(LocalDateTime toFix,MeasureResolution resolution) throws ParameterException {
switch(resolution) {
case MONTHLY : return toFix.with(lastDayOfMonth()).withHour(23).withMinute(59).withSecond(59).withNano(999999999);
default : throw new ParameterException("Invalid resolution "+resolution);
}
}
// private Instant fixLowerBound(Instant toFix,MeasureResolution resolution) throws ParameterException {
// switch(resolution) {
// case MONTHLY : {
// LocalDateTime local=LocalDateTime.ofInstant(toFix, ZoneId.of("UTC")).with(firstDayOfMonth());
// return toFix.with(ChronoField.DAY,local.getDayOfMonth()).with(ChronoField.CLOCK_HOUR_OF_DAY,0).
// with(ChronoField.MINUTE_OF_HOUR,0).with(ChronoField.SECOND_OF_MINUTE,0).with(ChronoField.NANO_OF_SECOND,0);
// }
// default : throw new ParameterException("Invalid resolution "+resolution);
// }
// }
//
// private Instant fixUpperBound(Instant toFix,MeasureResolution resolution) throws ParameterException {
// switch(resolution) {
// case MONTHLY : {
// LocalDateTime local=LocalDateTime.ofInstant(toFix, ZoneId.of("UTC")).with(lastDayOfMonth());
// return toFix.with(ChronoField.DAY_OF_MONTH,local.getDayOfMonth()).with(ChronoField.CLOCK_HOUR_OF_DAY,23).
// with(ChronoField.MINUTE_OF_HOUR,59).with(ChronoField.SECOND_OF_MINUTE,59).with(ChronoField.NANO_OF_SECOND,999999999);
// }
// default : throw new ParameterException("Invalid resolution "+resolution);
// }
// }
@Override
@ -240,7 +249,7 @@ public class AccountingDaoImpl implements AccountingDao{
//Context, Dim,time,measure
psMeasure.setString(1, context.getId());
psMeasure.setString(2, dim.getId());
psMeasure.setLong(3, record.getTime().getTime());
psMeasure.setTimestamp(3, new Timestamp(record.getTime().toEpochMilli()));
psMeasure.setLong(4, record.getMeasure());
psMeasure.setLong(5, record.getMeasure());
if(psMeasure.executeUpdate()==0) throw new SQLException("Error registering Measure : No inserted rows");
@ -257,12 +266,12 @@ public class AccountingDaoImpl implements AccountingDao{
private static final int getRangeSize(LocalDateTime from, LocalDateTime to, MeasureResolution resolution) throws ParameterException {
private static final long getRangeSize(Instant from, Instant to, MeasureResolution resolution) throws ParameterException {
log.debug("Evaluating time range between {} , {} [{}]",from,to,resolution);
Period p=Period.between(from.toLocalDate(), to.toLocalDate());
Period d=Period.between(LocalDateTime.ofInstant(from, UTC).toLocalDate(), LocalDateTime.ofInstant(to, UTC).toLocalDate());
switch(resolution) {
case MONTHLY : return p.getMonths()+(p.getYears()*12)+1; // +1 to include upper bound
case MONTHLY : return d.get(ChronoUnit.MONTHS)+(d.get(ChronoUnit.YEARS)*12)+1; // +1 to include upper bound
default : throw new ParameterException("Invalid resolution "+resolution);
}
@ -270,9 +279,9 @@ public class AccountingDaoImpl implements AccountingDao{
private static LocalDateTime increment(LocalDateTime toIncrement,MeasureResolution res,int offset){
private static Instant increment(Instant toIncrement,MeasureResolution res,int offset){
switch(res){
case MONTHLY : return toIncrement.plusMonths(offset);
case MONTHLY : return LocalDateTime.ofInstant(toIncrement, UTC).plus(offset,ChronoUnit.MONTHS).toInstant(ZoneOffset.UTC);
default : throw new RuntimeException("Unexpected Resolution "+res);
}
}
@ -289,25 +298,25 @@ public class AccountingDaoImpl implements AccountingDao{
private Series getSeries(Queries queries, LocalDateTime from, LocalDateTime to, Dimension dim, ScopeDescriptor scope, MeasureResolution res, int timeSlices) throws SQLException{
private Series getSeries(Queries queries, Instant from, Instant to, Dimension dim, ScopeDescriptor scope, MeasureResolution res, int timeSlices) throws SQLException{
Record[] records=new Record[timeSlices];
PreparedStatement ps=queries.prepareMeasuresByDimension(scope, res);
DateTimeFormatter formatter=getFormatter(res);
LocalDateTime toSetStartDate=from;
Instant currentTimeSlice=from;
for(int i=0;i<timeSlices;i++){
toSetStartDate=increment(from,res,i); // Increment Date
LocalDateTime toSetEndDate=increment(from,res,i+1);
currentTimeSlice=increment(from,res,i); // Increment Date
// Instant toSetEndDate=increment(from,res,i+1);
ps.setLong(1, toSetStartDate.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
ps.setLong(2, toSetEndDate.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
Timestamp t=new Timestamp(currentTimeSlice.toEpochMilli());
ps.setTimestamp(1, t);
ps.setTimestamp(2, t);
ps.setString(3, dim.getId());
ResultSet rs=ps.executeQuery();
Record toSet=new Record(formatter.format(toSetStartDate),0l);
Record toSet=new Record(formatter.format(LocalDateTime.ofInstant(currentTimeSlice, UTC)),0l);
if(rs.next()){
toSet.setY(rs.getLong(Measure.MEASURE));
}

View File

@ -4,8 +4,8 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.sql.Timestamp;
import java.time.Instant;
import org.gcube.accounting.accounting.summary.access.impl.DBStructure.CONTEXTS;
import org.gcube.accounting.accounting.summary.access.impl.DBStructure.DIMENSIONS;
@ -94,21 +94,23 @@ public class Queries {
* @return
* @throws SQLException
*/
public ResultSet getAvailableDimensions(LocalDateTime from, LocalDateTime to, ScopeDescriptor scope,MeasureResolution resolution) throws SQLException{
long fromTime=from.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
long toTime=to.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
public ResultSet getAvailableDimensions(Instant from, Instant to, ScopeDescriptor scope,MeasureResolution resolution) throws SQLException{
String query= String.format("Select * from %1$s where %2$s in (Select distinct(%3$s) from %4$s where %5$s AND %6$s)",
DIMENSIONS.TABLENAME, DIMENSIONS.ID,
Measure.DIMENSION,Measure.TABLENAME,
Measure.TIME+">=? AND "+Measure.TIME+"<=?",
String.format("EXTRACT(YEAR FROM %1$s) >= EXTRACT(YEAR FROM CAST (? AS TIMESTAMP)) AND EXTRACT (MONTH FROM %1$s)>= EXTRACT(MONTH FROM CAST (? AS TIMESTAMP)) "
+ "AND EXTRACT(YEAR FROM %1$s) <= EXTRACT(YEAR FROM CAST (? AS TIMESTAMP)) AND EXTRACT (MONTH FROM %1$s)<= EXTRACT(MONTH FROM CAST (? AS TIMESTAMP)) ", Measure.TIME),
// "true",
Measure.CONTEXT+" IN "+asIDSet(scope));
PreparedStatement toReturn= conn.prepareStatement(query);
toReturn.setLong(1, fromTime);
toReturn.setLong(2, toTime);
toReturn.setTimestamp(1, new Timestamp(from.toEpochMilli()));
toReturn.setTimestamp(2, new Timestamp(from.toEpochMilli()));
toReturn.setTimestamp(3, new Timestamp(to.toEpochMilli()));
toReturn.setTimestamp(4, new Timestamp(to.toEpochMilli()));
log.debug("Performing query {} with params {} and {} ",query,fromTime,toTime);
log.debug("Performing query {} ",query);
return toReturn.executeQuery();
}
@ -117,8 +119,8 @@ public class Queries {
* Prepares a statement for Getting Dim=? in time interval for the scope set
*
* PS params :
* 1- long from
* 2- long to
* 1- long current slice of time
* 2- long current slice of time
* 3- String dimension
*
*
@ -136,7 +138,7 @@ public class Queries {
return conn.prepareStatement(String.format("Select sum(%1$s) as %1$s from %2$s where %3$s AND %4$s AND %5$s=? group by %6$s order by %7$s",
Measure.MEASURE,Measure.TABLENAME,
Measure.CONTEXT+" IN "+asIDSet(scope), //context ok
Measure.TIME+">=? AND "+Measure.TIME+"<=?", // time ok, PS Parameter
String.format("EXTRACT(YEAR FROM %1$s) = EXTRACT(YEAR FROM CAST (? AS TIMESTAMP)) AND EXTRACT (MONTH FROM %1$s)= EXTRACT(MONTH FROM CAST (? AS TIMESTAMP)) ",Measure.TIME), // time ok, PS Parameter
Measure.DIMENSION, // dimension ok, PS parameter
Measure.TIME, // group by (time)
Measure.TIME)); // order by time
@ -145,6 +147,8 @@ public class Queries {
}
public static final String asIDSet(ScopeDescriptor desc){
return "("+scopeList(desc)+")";
}

View File

@ -1,6 +1,6 @@
package org.gcube.accounting.accounting.summary.access.model.update;
import java.util.Date;
import java.time.Instant;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
@ -17,7 +17,7 @@ import lombok.ToString;
public class AccountingRecord {
private ScopeDescriptor context;
private Date time;
private Instant time;
private Dimension dimension;
private Long measure;
}

View File

@ -8,15 +8,16 @@ public class DummyContextTreeProvider implements ContextTreeProvider {
@Override
public ScopeDescriptor getTree(Object context) {
ScopeDescriptor toReturn=new ScopeDescriptor("Portal", "portale");
ScopeDescriptor group1=new ScopeDescriptor("Group1","group1");
group1.getChildren().add(new ScopeDescriptor("BlueBridge Project", "/d4science.research-infrastructures.eu/gCubeApps/BlueBridgeProject"));
ScopeDescriptor toReturn=new ScopeDescriptor("Portal", "20508");
ScopeDescriptor group1=new ScopeDescriptor("D4Science labs","group1");
group1.getChildren().add(new ScopeDescriptor("Analytics", "/d4science.research-infrastructures.eu/D4Research/AnalyticsLab"));
// group1.getChildren().add(new ScopeDescriptor("Infra Training", "/d4science.research-infrastructures.eu/D4Research/InfraTraining"));
// group1.getChildren().add(new ScopeDescriptor("RProto", "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"));
toReturn.getChildren().add(group1);
ScopeDescriptor group2=new ScopeDescriptor("Group2","group2");
group2.getChildren().add(new ScopeDescriptor("Biodiversity Lab","/d4science.research-infrastructures.eu/gCubeApps/BiodiversityLab"));
group2.getChildren().add(new ScopeDescriptor("IOTC SS3","/d4science.research-infrastructures.eu/gCubeApps/IOTC_SS3"));
toReturn.getChildren().add(group2);
toReturn.getChildren().add(new ScopeDescriptor("Fao Tuna Atlas","/d4science.research-infrastructures.eu/gCubeApps/FAO_TunaAtlas"));
// group2.getChildren().add(new ScopeDescriptor("AGINFRA","/d4science.research-infrastructures.eu/D4Research/AGINFRAplus"));
// group2.getChildren().add(new ScopeDescriptor("Tag Me","/d4science.research-infrastructures.eu/SoBigData/TagMe"));
toReturn.getChildren().add(group2);
return toReturn;
}
}

View File

@ -1,35 +1,42 @@
package org.gcube.data.access.accounting.summary.access.test;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Date;
import java.util.GregorianCalendar;
import org.gcube.accounting.accounting.summary.access.AccountingDao;
import org.gcube.accounting.accounting.summary.access.ParameterException;
import org.gcube.accounting.accounting.summary.access.impl.AccountingDaoImpl;
import org.gcube.accounting.accounting.summary.access.impl.Queries;
import org.gcube.accounting.accounting.summary.access.model.MeasureResolution;
import org.gcube.accounting.accounting.summary.access.model.Report;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
public class SimpleTest {
private static final SimpleDateFormat monthFormatter=new SimpleDateFormat("yyyy-MM-dd-HH:mm ZZZ");
public static void main(String[] args) throws ParameterException, Exception {
TokenSetter.set("/d4science.research-infrastructures.eu");
TokenSetter.set("/gcube");
// AccountingDao dao=AccountingDao.get();
AccountingDao dao=AccountingDao.get(new DummyContextTreeProvider());
ScopeDescriptor desc=dao.getTree(null);
System.out.println(desc);
System.out.println("1527811200000 "+monthFormatter.format(new Date(1527811200000l)));
System.out.println("1527804000000 "+monthFormatter.format(new Date(1527804000000l)));
// Date from=new Date(1514764800000l);
Date from=new GregorianCalendar(2017,0,1).getTime();
Date to=new GregorianCalendar(2018,7,1).getTime();
Instant from=Instant.parse("2018-01-03T10:15:30.00Z");
// Date to=from;
// System.out.println("Time: " +from.toEpochMilli()+" -> "+monthFormatter.format(from));
// Date to=new GregorianCalendar(2018,6,1).getTime();
Instant to=Instant.parse("2018-07-03T10:15:30.00Z");
scan(desc,from,to,dao);
@ -38,7 +45,7 @@ public class SimpleTest {
}
private static void scan(ScopeDescriptor desc, Date from, Date to, AccountingDao dao) throws ParameterException, SQLException {
private static void scan(ScopeDescriptor desc, Instant from, Instant to, AccountingDao dao) throws ParameterException, SQLException {
System.out.println("**********************************************************************************************************");
System.out.println(desc);
System.out.println(dao.getReportByScope(desc, from, to, MeasureResolution.MONTHLY));

View File

@ -1,8 +1,7 @@
package org.gcube.data.access.accounting.summary.access.test;
import java.sql.SQLException;
import java.util.Date;
import java.util.GregorianCalendar;
import java.time.Instant;
import org.gcube.accounting.accounting.summary.access.AccountingDao;
import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
@ -20,7 +19,7 @@ public class WriteTest {
ScopeDescriptor devsec=new ScopeDescriptor("Devsec","/gcube/devsec");
Dimension dummyDimension=new Dimension("dummy_xyz","Dummy dimension",null,"Test Measures");
Date measureDate=new GregorianCalendar().getTime();
Instant measureDate=Instant.parse("2018-01-03T10:15:30.00Z");
AccountingRecord[] records=new AccountingRecord[] {
new AccountingRecord(devsec, measureDate, dummyDimension, 123l)

View File

@ -6,7 +6,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import java.sql.Timestamp;
import java.util.HashMap;
import org.gcube.accounting.accounting.summary.access.impl.DBStructure.CONTEXTS;
@ -30,16 +30,16 @@ public class Importer {
}
private static Connection getDestinationConn() throws SQLException, ClassNotFoundException{
// //DEV OLD DB
// String user="analytics_b_dev_u";
// String password ="78cb625303be21b";
// String url ="jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/analytics_b_dev";
//
//DEV OLD DB
String user="analytics_b_dev_u";
String password ="78cb625303be21b";
String url ="jdbc:postgresql://postgresql-srv-dev.d4science.org:5432/analytics_b_dev";
// PRODUCTION OLD DB
String user="analytics_board_u";
String password ="b52b64ab07ea0b5";
String url ="jdbc:postgresql://postgresql-srv.d4science.org:5432/analytics_board";
// // PRODUCTION OLD DB
// String user="analytics_board_u";
// String password ="b52b64ab07ea0b5";
// String url ="jdbc:postgresql://postgresql-srv.d4science.org:5432/analytics_board";
Class.forName("org.postgresql.Driver");
@ -78,7 +78,7 @@ public class Importer {
createStatment.execute("CREATE TABLE IF NOT EXISTS "+Measure.TABLENAME+
"("+Measure.MEASURE+" bigint NOT NULL, "
+ Measure.TIME+" bigint NOT NULL,"
+ Measure.TIME+" timestamp with time zone NOT NULL,"
+ Measure.CONTEXT+" varchar (125) NOT NULL,"
+ Measure.DIMENSION+" varchar (125) NOT NULL,"
+"CONSTRAINT "+Measure.TABLENAME+"_pKey PRIMARY KEY ("+Measure.TIME+","+Measure.CONTEXT+","+Measure.DIMENSION+"),"
@ -126,8 +126,8 @@ public class Importer {
String contextLabel= sourceRS.getString("context_label");
String dimension = sourceRS.getString("dimension");
String dimensionLabel= dimension;
Date day=sourceRS.getDate("day");
// Date day=sourceRS.getDate("day");
Timestamp instant=new Timestamp(sourceRS.getDate("day").getTime());
@ -152,7 +152,7 @@ public class Importer {
measurePs.setString(1, context);
measurePs.setString(2, dimension);
measurePs.setLong(3, day.getTime());
measurePs.setTimestamp(3, instant);
measurePs.setLong(4, measure);
measurePs.setLong(5, measure);
if(measurePs.executeUpdate()==0) throw new Exception("No inserted Measure");