This commit is contained in:
Fabio Sinibaldi 2019-02-01 17:45:25 +00:00
parent 965b035c31
commit ab561d96a4
9 changed files with 195 additions and 115 deletions

View File

@ -20,7 +20,7 @@
<properties>
<webappDirectory>${project.basedir}/src/main/webapp/WEB-INF</webappDirectory>
<distroDirectory>${project.basedir}/distro</distroDirectory>
<jersey-version>2.13</jersey-version>
<!-- <jersey-version>2.13</jersey-version> -->
</properties>
<dependencyManagement>

View File

@ -1,10 +1,10 @@
package org.gcube.application.perform.service.engine.impl;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.gcube.application.perform.service.engine.model.CSVExportRequest;
import org.gcube.application.perform.service.engine.model.DBField;
@ -13,19 +13,20 @@ public class ExportCSVQuery extends Query {
private Map<String,Map<String,String>> mappings=new HashMap<>();
private String fieldList;
private String tablename;
private CSVExportRequest theRequest;
private SchemaDefinition schema;
private Map<String,DBField> actualStructure;
public ExportCSVQuery(String query, DBField[] fields, CSVExportRequest theRequest, SchemaDefinition schema, Map<String,DBField> actualStructure) {
super(query, fields);
this.theRequest=theRequest;
this.schema=schema;
this.actualStructure=actualStructure;
}
@ -34,115 +35,145 @@ public class ExportCSVQuery extends Query {
}
public void setTablename(String tablename) {
this.tablename = tablename;
}
@Override
public String getQuery() {
String selectedFields=replaceWithMappings(getFieldList());
String conditionString =getConditionString();
if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
return String.format("SELECT %1$s FROM %2$s %3$s",
selectedFields, tablename, conditionString);
}
public String getQueryForMappedFields(String filterMappingKey,String...fields) {
StringBuilder b=new StringBuilder();
for(String f:fields)
b.append(f+",");
b.setLength(b.lastIndexOf(","));
log.debug("Creating query for fields {} against table {} ",b,tablename);
String selectedFields=replaceWithMappings(b.toString());
String condition=getFilterWithMapping(filterMappingKey);
return String.format("SELECT %1$s FROM %2$s %3$s",
selectedFields, tablename, condition);
}
private String getFilterWithMapping(String mappingFilterKey) {
StringBuilder conditionBuilder=new StringBuilder("WHERE ");
if(mappings.containsKey(mappingFilterKey)&&(!mappings.get(mappingFilterKey).isEmpty())) {
String actualField=actualField(mappingFilterKey);
log.debug("Setting filter By Mappings for field {}, size {} ",actualField,mappings.get(mappingFilterKey).size());
for(Entry<String,String> mappingFilter:mappings.get(mappingFilterKey).entrySet()) {
conditionBuilder.append(String.format("%1$s = '%2$s' OR", actualField,mappingFilter.getKey()));
}
conditionBuilder.setLength(conditionBuilder.lastIndexOf("OR"));
// Add selection filter..
String filteringCondition=getConditionString();
if(filteringCondition.length()>0) conditionBuilder.append(" AND "+filteringCondition);
}else {
log.debug("No mappings to search For ");
conditionBuilder.append("FALSE");
}
return conditionBuilder.toString();
}
private String replaceWithMappings(String selectionFields) {
String toReturn=selectionFields;
// fieldLabel -> (uuid->name)
for(Entry<String,Map<String,String>> mapping: mappings.entrySet()) {
if(exists(mapping.getKey())) {
String actualMapped=actualField(mapping.getKey());
if(toReturn.contains(actualMapped)) {
StringBuilder caseBuilder=new StringBuilder("CASE "+actualMapped);
for(Entry<String,String> condition: mapping.getValue().entrySet())
caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
caseBuilder.append("END AS "+actualMapped);
toReturn=toReturn.replace(actualMapped, caseBuilder.toString());
}
}
}
return toReturn.toString();
}
private String getConditionString() {
ArrayList<String> orGroups=new ArrayList<String>();
// AREA
if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && actualStructure.containsKey(schema.getAreaField())) {
String areaField=actualStructure.get(schema.getAreaField()).getFieldName();
StringBuilder areas=new StringBuilder();
for(String area:theRequest.getAreas()) {
areas.append(String.format("%1$s= '%2$s' OR", areaField,area));
}
orGroups.add(areas.substring(0,areas.lastIndexOf("OR")));
}
if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && exists(schema.getAreaField()))
orGroups.add(getFilterByMultipleValues(theRequest.getAreas(), schema.getAreaField()));
// QUARTER
if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && actualStructure.containsKey(schema.getQuarterField())) {
String quarterField=actualStructure.get(schema.getQuarterField()).getFieldName();
StringBuilder quarterString=new StringBuilder();
for(String q:theRequest.getQuarters()) {
quarterString.append(String.format("%1$s= '%2$s' OR", quarterField,q));
}
orGroups.add(quarterString.substring(0,quarterString.lastIndexOf("OR")));
}
if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && exists(schema.getQuarterField()))
orGroups.add(getFilterByMultipleValues(theRequest.getQuarters(), schema.getQuarterField()));
// SPECIES ID
if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && actualStructure.containsKey(schema.getSpeciesField())) {
String speciesField=actualStructure.get(schema.getSpeciesField()).getFieldName();
StringBuilder speciesString=new StringBuilder();
for(String s:theRequest.getSpeciesIds()) {
speciesString.append(String.format("%1$s= '%2$s' OR", speciesField,s));
}
orGroups.add(speciesString.substring(0,speciesString.lastIndexOf("OR")));
}
if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && exists(schema.getSpeciesField()))
orGroups.add(getFilterByMultipleValues(theRequest.getSpeciesIds(), schema.getSpeciesField()));
// PERIOD
if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && actualStructure.containsKey(schema.getPeriodField())) {
String periodField=actualStructure.get(schema.getPeriodField()).getFieldName();
StringBuilder periodString=new StringBuilder();
for(String p:theRequest.getPeriods()) {
periodString.append(String.format("%1$s= '%2$s' OR", periodField,p));
}
orGroups.add(periodString.substring(0,periodString.lastIndexOf("OR")));
}
if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && exists(schema.getPeriodField()))
orGroups.add(getFilterByMultipleValues(theRequest.getPeriods(), schema.getPeriodField()));
StringBuilder toReturn=new StringBuilder("");
for(String orGroup:orGroups) {
toReturn.append(orGroup+ " AND ");
}
if(toReturn.length()>0)
return toReturn.substring(0, toReturn.lastIndexOf("AND"));
else return toReturn.toString();
toReturn.setLength(toReturn.lastIndexOf("AND"));
return toReturn.toString();
}
public void setTablename(String tablename) {
this.tablename = tablename;
}
public void setFieldList(Collection<DBField> fields) {
private String getFieldList() {
StringBuilder b=new StringBuilder();
for(DBField f:fields)
b.append(f.getFieldName()+",");
fieldList=b.toString().substring(0,b.lastIndexOf(","));
for(DBField f:actualStructure.values())
b.append(f.getFieldName()+",");
return b.toString().substring(0,b.lastIndexOf(","));
}
@Override
public String getQuery() {
String selectedFields=fieldList;
for(Entry<String,Map<String,String>> mapping:mappings.entrySet()) {
StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey());
for(Entry<String,String> condition: mapping.getValue().entrySet())
caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
caseBuilder.append("END AS "+mapping.getKey()+",");
selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString());
}
String conditionString =getConditionString();
if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
return String.format("SELECT %1$s FROM %2$s %3$s",
selectedFields, tablename, conditionString);
private String actualField(String label) {
return actualStructure.get(label).getFieldName();
}
public String getQueryForFields(String...fields ) {
StringBuilder b=new StringBuilder();
for(String f:fields)
b.append(f+",");
String selectedFields=b.toString().substring(0,b.lastIndexOf(","));
for(Entry<String,Map<String,String>> mapping:mappings.entrySet()) {
StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey());
for(Entry<String,String> condition: mapping.getValue().entrySet())
caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
caseBuilder.append("END AS "+mapping.getKey()+",");
selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString());
}
private boolean exists(String label) {
return actualStructure.containsKey(label);
}
String conditionString =getConditionString();
if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
return String.format("SELECT %1$s FROM %2$s %3$s",
selectedFields, tablename, conditionString);
private String getFilterByMultipleValues(Set<String> filterValues,String filterLabel) {
String field=actualField(filterLabel);
StringBuilder toReturn=new StringBuilder();
for(String p:filterValues)
toReturn.append(String.format("%1%s = '%2$s' OR",field,p));
toReturn.setLength(toReturn.lastIndexOf("OR"));
return toReturn.toString();
}
}

View File

@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
public class Query {
private static final Logger log= LoggerFactory.getLogger(Query.class);
protected static final Logger log= LoggerFactory.getLogger(Query.class);
protected final String query;
protected final ArrayList<DBField> psFields;

View File

@ -74,6 +74,12 @@ public class CSVExportRequest {
periods.addAll(toAdd);
return this;
}
@Override
public String toString() {
return "CSVExportRequest [type=" + type + ", farmIds=" + farmIds + ", quarters=" + quarters + ", areas=" + areas
+ ", speciesIds=" + speciesIds + ", periods=" + periods + "]";
}
}

View File

@ -19,7 +19,6 @@ import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.gcube.application.perform.service.engine.DataBaseManager;
import org.gcube.application.perform.service.engine.impl.ExportCSVQuery;
import org.gcube.application.perform.service.engine.impl.ImporterImpl;
import org.gcube.application.perform.service.engine.impl.Queries;
import org.gcube.application.perform.service.engine.impl.Query;
import org.gcube.application.perform.service.engine.impl.SchemaDefinition;
@ -36,7 +35,12 @@ import org.slf4j.LoggerFactory;
public class ImportedTable {
private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class);
@Override
public String toString() {
return "ImportedTable [tablename=" + tablename + "]";
}
private static final Logger log= LoggerFactory.getLogger(ImportedTable.class);
/**
* CSV FILE labels -> DBField
@ -213,7 +217,6 @@ public class ImportedTable {
try {
ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels);
exportQuery.setFieldList(labels.values());
exportQuery.setTablename(tablename);
Map<String,String> farmMapping=new HashMap<>();
@ -241,14 +244,22 @@ public class ImportedTable {
}
}
if(schema.getAssociationUUIDField()!=null)
if(schema.getAssociationUUIDField()!=null) {
log.debug("Setting Association Mapping : "+associationMapping);
exportQuery.setMapping(schema.getAssociationUUIDField(), associationMapping);
if(schema.getCompanyUUIDField()!=null)
}
if(schema.getCompanyUUIDField()!=null) {
log.debug("Setting Company Mapping : "+companyMapping);
exportQuery.setMapping(schema.getCompanyUUIDField(), companyMapping);
if(schema.getFarmUUIDField()!=null)
}
if(schema.getFarmUUIDField()!=null) {
log.debug("Setting Farm Mapping : "+farmMapping);
exportQuery.setMapping(schema.getFarmUUIDField(), farmMapping);
if(schema.getBatchUUIDField()!=null)
}
if(schema.getBatchUUIDField()!=null) {
log.debug("Setting Batch Mapping : "+batchMapping);
exportQuery.setMapping(schema.getBatchUUIDField(), batchMapping);
}
log.trace("Performing actual query towards {} ",tablename);
@ -273,7 +284,12 @@ public class ImportedTable {
for(String label:personalLabels) {
toExtractFields.add(labels.get(label).getFieldName());
}
String sqlPersonal=exportQuery.getQueryForFields(toExtractFields.toArray(new String[toExtractFields.size()]));
String sqlPersonal=exportQuery.getQueryForMappedFields(schema.getFarmUUIDField(),
toExtractFields.toArray(new String[toExtractFields.size()]));
log.debug("Query is {} ",sqlPersonal);
csvRs=stmt.executeQuery(sqlPersonal);

View File

@ -3,18 +3,15 @@ package org.gcube.application.perform.service;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.file.Paths;
import java.sql.ResultSet;
import java.sql.Connection;
import java.sql.SQLException;
import org.gcube.application.perform.service.engine.DataBaseManager;
import org.gcube.application.perform.service.engine.PerformanceManager;
import org.gcube.application.perform.service.engine.dm.DMException;
import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl;
import org.gcube.application.perform.service.engine.impl.Queries;
import org.gcube.application.perform.service.engine.model.DBField;
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
import org.gcube.application.perform.service.engine.model.InternalException;
import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
public class LoadSchemaTest {
@ -36,13 +33,26 @@ public class LoadSchemaTest {
* sourceUrl=https://data1-d.d4science.org/shub/E_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA==, sourceVersion=1.1, startTime=2019-01-25T14:52:31.442Z, endTime=null, status=ACCEPTED, lock=localhost, caller=MSgXVCkHb0SDoQLlCLQV9Kj8dPJlJ6gY+XicZJhenQkyuxA11kGUIdhxKs3jUdGK, computationId=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationUrl=http://dataminer1-pre.d4science.org:80//wps/RetrieveResultServlet?id=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationOperator=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR, computationOperatorName=Performfish Data Extractor, computationRequest=https://dataminer1-pre.d4science.org/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token=***REMOVED***&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR&DataInputs=InputData=https%3A%2F%2Fdata1-d.d4science.org%2Fshub%2FE_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA%3D%3D;BatchType=HATCHERY_INDIVIDUAL;FarmID=12682101;]
*/
// RELOAD DATA FROM COMPLETED ROUTINES
// ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(),
// new DBQueryDescriptor().
// add(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.STATUS),"COMPLETE")).executeQuery();
//
// while(rs.next()) {
//
// ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs);
// // RELOAD DATA
// mng.loadOutputData(desc);
// // RELAUNCH REQUEST
//
//
// }
ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(), new DBQueryDescriptor(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.ID),26l)).executeQuery();
rs.next();
Queries.rowToDescriptor(rs);
ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs);
// CLEAN IMPORTS
Connection conn = DataBaseManager.get().getConnection();
conn.createStatement().executeUpdate("DELETE FROM "+DBField.ImportRoutine.TABLE);
conn.commit();
mng.loadOutputData(desc);

View File

@ -1,9 +1,16 @@
package org.gcube.application.perform.service;
import java.util.HashMap;
import java.util.Map;
import javax.ws.rs.core.MultivaluedHashMap;
import org.gcube.application.perform.service.engine.model.DBField;
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
import org.glassfish.jersey.internal.util.collection.ImmutableMultivaluedMap;
import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractor;
import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractorProvider;
import org.glassfish.jersey.server.model.Parameter;
public class MiscTests {
@ -13,6 +20,14 @@ public class MiscTests {
condition.put(DBField.Farm.fields.get(DBField.Farm.FARM_ID), "suca");
condition.put(DBField.Farm.fields.get(DBField.Farm.UUID), "boh");
System.out.println(new DBQueryDescriptor(condition));
// theMap.put("farmid", "128");
// ImmutableMultivaluedMap<String, String> map=new ImmutableMultivaluedMap<>(theMap);
//
}
}

View File

@ -14,11 +14,11 @@ public class PerformanceTest extends CommonTest{
public void getPerformance() {
WebTarget target=
target(ServiceConstants.Performance.PATH).
queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "1233556","12346").
// queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
// queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
// queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
// queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "13625424","1233556","12346").
queryParam(ServiceConstants.Performance.BATCH_TYPE_PARAMETER, "GROW_OUT_AGGREGATED");
System.out.println(target.getUri());

View File

@ -8,7 +8,9 @@
</encoder>
</appender>
<root level="debug">
<logger name="org.gcube.application" level="DEBUG"/>
<root level="WARN">
<appender-ref ref="STDOUT" />
</root>
</configuration>