gCubeSystem
/
perform-service_broken
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed mapping on SNAPSHOT

This commit is contained in:
Fabio Sinibaldi 2020-01-29 15:35:02 +01:00
parent c9f455ee4c
commit 49dbf76b8e
8 changed files with 58 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/gcube/application/perform/service/LocalConfiguration.java
View File

@ -28,6 +28,10 @@ public class LocalConfiguration {
public static final String COMMIT_SCHEMA="schema.load.commit";
public static final String DEANONIMIZATION_INCLUDE_SNAPSHOT="deanonimization.default_include_SNAPSHOT";
public static final String PERFORMANCE_DEFAULT_INCLUDE_SNAPSHOT="performance.default_include_SNAPSHOT";
static LocalConfiguration instance=null;

3
src/main/java/org/gcube/application/perform/service/ServiceConstants.java
View File

@ -39,7 +39,8 @@ public interface ServiceConstants {
public static final String BATCH_TYPE_PARAMETER="batch_type";
public static final String PERIOD_PARAMETER="period";
public static final String HISTORIC="historic";
public static final String INCLUDE_SNAPSHOST="include_snapshots";
public static final String DEANONIMIZE_SNAPSHOST="deanonimize_snapshots";
public static final String START_YEAR="start_year";

18
src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java
View File

@ -22,9 +22,9 @@ public class ExportCSVQuery extends Query {
private ArrayList<String> exportCSVFieldOrder;
public ExportCSVQuery(String query, DBField[] queryParams, CSVExportRequest theRequest,
public ExportCSVQuery(CSVExportRequest theRequest,
SchemaDefinition schema, Map<String,DBField> actualStructure, ArrayList<String> exportFieldsOrder) {
super(query, queryParams);
super("", null);
this.theRequest=theRequest;
this.schema=schema;
this.actualStructure=actualStructure;
@ -33,6 +33,8 @@ public class ExportCSVQuery extends Query {
}
public void setMapping(String field, Map<String,String> mapping) {
mappings.put(field, mapping);
@ -64,6 +66,9 @@ public class ExportCSVQuery extends Query {
StringBuilder b=new StringBuilder();
for(String f:fields)
b.append(f+",");
if(schema.getSnapshotFieldName()!=null)
b.append(schema.getSnapshotFieldName()+",");
b.setLength(b.lastIndexOf(","));
log.debug("Creating query for fields {} against table {} ",b,tablename);
@ -146,6 +151,13 @@ public class ExportCSVQuery extends Query {
if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && exists(schema.getPeriodField()))
orGroups.add(getFilterByMultipleValues(theRequest.getPeriods(), schema.getPeriodField()));
// SNAPSHOTS
if(!theRequest.getIncludeHistoric())
if(schema.getSnapshotFieldName()!=null && exists(schema.getSnapshotFieldName()))
orGroups.add(schema.getSnapshotFieldName()+" = "+theRequest.getIncludeHistoric());
StringBuilder toReturn=new StringBuilder("");
for(String orGroup:orGroups) {
toReturn.append("("+orGroup+") AND ");
@ -166,6 +178,8 @@ public class ExportCSVQuery extends Query {
DBField f = actualStructure.get(label);
b.append(f.getFieldName()+",");
}
if(schema.getSnapshotFieldName()!=null)
b.append(schema.getSnapshotFieldName()+",");
return b.toString().substring(0,b.lastIndexOf(","));
}

10
src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java
View File

@ -20,10 +20,20 @@ public class CSVExportRequest {
private Boolean includeHistoric=false;
private Integer startYear=0;
private Boolean deanonimizeSNAPSHOTS=false;
public Boolean getIncludeHistoric() {
return includeHistoric;
}
public Boolean getDeanonimizeSNAPSHOTS() {
return deanonimizeSNAPSHOTS;
}
public void setDeanonimizeSNAPSHOTS(Boolean deanonimizeSNAPSHOTS) {
this.deanonimizeSNAPSHOTS = deanonimizeSNAPSHOTS;
}
public Integer getStartYear() {
return startYear;
}

8
src/main/java/org/gcube/application/perform/service/engine/model/importer/CSVRecordConverter.java
View File

@ -22,7 +22,7 @@ public class CSVRecordConverter {
}
private Map<String,Map<String,String>> mappings=new HashMap<>();
private Map<DBField,Set<Object>> condition=new HashMap<>();
private Map<DBField,Set<? extends Object>> condition=new HashMap<>();
// private Set<String> conditionValues=new HashSet<>();
private boolean isAlwaysMap=false;
@ -55,9 +55,9 @@ public class CSVRecordConverter {
// }
public void addCondition(DBField conditionField,Set<Object> values) {
public void addCondition(DBField conditionField,Set<? extends Object> values) {
String label=labels.get(conditionField.getFieldName()).getFieldName();
DBField labelledField=new DBField(conditionField.getType(),label.substring(1,label.length()-1));
DBField labelledField=new DBField(conditionField.getType(),label.replace("\"",""));
this.condition.put(labelledField, values);
}
@ -76,7 +76,7 @@ public class CSVRecordConverter {
// DefaultBehaviour
if(condition.isEmpty()) return false;
else {
for(Entry<DBField,Set<Object>> entry:condition.entrySet()) {
for(Entry<DBField,Set<? extends Object>> entry:condition.entrySet()) {
Object obj=rs.getObject(entry.getKey().getFieldName());
if(!entry.getValue().contains(obj)) return false;
}

16
src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java
View File

@ -14,6 +14,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -276,7 +277,8 @@ public class ImportedTable {
try {
CSVRecordConverter queryConverter=new CSVRecordConverter(labels);
ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels,csvFields);
ExportCSVQuery exportQuery=new ExportCSVQuery(request,schema,labels,csvFields);
exportQuery.setTablename(tablename);
Map<String,String> farmMapping=new HashMap<>();
@ -328,12 +330,14 @@ public class ImportedTable {
}
// Set mapping condition
if(schema.getFarmUUIDField()!=null)
queryConverter.addCondition(new DBField(Types.VARCHAR,schema.getFarmUUIDField()), Collections.singleton(farmMapping.keySet()));
if(hasSNAPSHOT())
queryConverter.addCondition(getSNAPSHOTField(), Collections.singleton(request.getIncludeHistoric()));
if(schema.getFarmUUIDField()!=null) {
queryConverter.addCondition(new DBField(Types.VARCHAR,schema.getFarmUUIDField()), farmMapping.keySet());
}
if(hasSNAPSHOT()) {
if(!request.getDeanonimizeSNAPSHOTS())
queryConverter.addCondition(getSNAPSHOTField(), Collections.singleton(false));
}
log.trace("Performing actual query towards {} ",tablename);

10
src/main/java/org/gcube/application/perform/service/rest/Performance.java
View File

@ -16,6 +16,7 @@ import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.gcube.application.perform.service.LocalConfiguration;
import org.gcube.application.perform.service.PerformServiceManager;
import org.gcube.application.perform.service.ServiceConstants;
import org.gcube.application.perform.service.engine.PerformanceManager;
@ -69,10 +70,17 @@ public class Performance {
request.addSpecies(InterfaceCommons.getParameter(parameters, ServiceConstants.Performance.SPECIES_ID_PARAMETER, false));
request.addPeriods(InterfaceCommons.getParameter(parameters, ServiceConstants.Performance.PERIOD_PARAMETER, false));
request.setIncludeHistoric(Boolean.valueOf(InterfaceCommons.getParamOrDefault(parameters, ServiceConstants.Performance.HISTORIC, "true")));
request.setIncludeHistoric(
Boolean.valueOf(
InterfaceCommons.getParamOrDefault(parameters, ServiceConstants.Performance.INCLUDE_SNAPSHOST,
LocalConfiguration.getProperty(LocalConfiguration.PERFORMANCE_DEFAULT_INCLUDE_SNAPSHOT))));
request.setStartYear(Integer.valueOf(InterfaceCommons.getParamOrDefault(parameters, ServiceConstants.Performance.START_YEAR, "1900")));
request.setDeanonimizeSNAPSHOTS(Boolean.valueOf(
InterfaceCommons.getParamOrDefault(parameters, ServiceConstants.Performance.DEANONIMIZE_SNAPSHOST,
LocalConfiguration.getProperty(LocalConfiguration.DEANONIMIZATION_INCLUDE_SNAPSHOT))));
log.debug("Export request : {} ",request);
return perform.generateCSV(request);

3
src/main/webapp/WEB-INF/config.properties
View File

@ -11,4 +11,7 @@ dm.importer.computationid=org.gcube.dataanalysis.wps.statisticalmanager.synchser
schema.load=true
schema.load.skipError=true
schema.load.commit=true
deanonimization.default_include_SNAPSHOT=false
performance.default_exclude_SNAPSHOT=false