From ab561d96a4f6388b6a7efdfdcaa3673f7a676046 Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Fri, 1 Feb 2019 17:45:25 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/application/perform-service@176944 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 2 +- .../service/engine/impl/ExportCSVQuery.java | 211 ++++++++++-------- .../perform/service/engine/impl/Query.java | 2 +- .../engine/model/CSVExportRequest.java | 6 + .../engine/model/importer/ImportedTable.java | 32 ++- .../perform/service/LoadSchemaTest.java | 28 ++- .../perform/service/MiscTests.java | 15 ++ .../perform/service/PerformanceTest.java | 10 +- src/test/resources/logback.xml | 4 +- 9 files changed, 195 insertions(+), 115 deletions(-) diff --git a/pom.xml b/pom.xml index b0e3d3d..6b3ffd2 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ ${project.basedir}/src/main/webapp/WEB-INF ${project.basedir}/distro - 2.13 + diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java b/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java index 99b61ae..759eb23 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java +++ b/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java @@ -1,10 +1,10 @@ package org.gcube.application.perform.service.engine.impl; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.gcube.application.perform.service.engine.model.CSVExportRequest; import org.gcube.application.perform.service.engine.model.DBField; @@ -13,19 +13,20 @@ public class ExportCSVQuery extends Query { private Map> mappings=new HashMap<>(); - private String fieldList; private String tablename; private CSVExportRequest theRequest; private SchemaDefinition schema; private Map actualStructure; - + public ExportCSVQuery(String query, DBField[] fields, CSVExportRequest theRequest, SchemaDefinition schema, Map actualStructure) { super(query, fields); this.theRequest=theRequest; this.schema=schema; this.actualStructure=actualStructure; + + } @@ -34,115 +35,145 @@ public class ExportCSVQuery extends Query { } + + + public void setTablename(String tablename) { + this.tablename = tablename; + } + + + @Override + public String getQuery() { + + String selectedFields=replaceWithMappings(getFieldList()); + + + String conditionString =getConditionString(); + if(conditionString.length()>0) conditionString= "WHERE "+conditionString; + + + return String.format("SELECT %1$s FROM %2$s %3$s", + selectedFields, tablename, conditionString); + } + + public String getQueryForMappedFields(String filterMappingKey,String...fields) { + StringBuilder b=new StringBuilder(); + for(String f:fields) + b.append(f+","); + b.setLength(b.lastIndexOf(",")); + + log.debug("Creating query for fields {} against table {} ",b,tablename); + + String selectedFields=replaceWithMappings(b.toString()); + String condition=getFilterWithMapping(filterMappingKey); + + + + return String.format("SELECT %1$s FROM %2$s %3$s", + selectedFields, tablename, condition); + } + + private String getFilterWithMapping(String mappingFilterKey) { + + StringBuilder conditionBuilder=new StringBuilder("WHERE "); + if(mappings.containsKey(mappingFilterKey)&&(!mappings.get(mappingFilterKey).isEmpty())) { + String actualField=actualField(mappingFilterKey); + log.debug("Setting filter By Mappings for field {}, size {} ",actualField,mappings.get(mappingFilterKey).size()); + + + for(Entry mappingFilter:mappings.get(mappingFilterKey).entrySet()) { + conditionBuilder.append(String.format("%1$s = '%2$s' OR", actualField,mappingFilter.getKey())); + } + conditionBuilder.setLength(conditionBuilder.lastIndexOf("OR")); + + // Add selection filter.. + String filteringCondition=getConditionString(); + if(filteringCondition.length()>0) conditionBuilder.append(" AND "+filteringCondition); + }else { + log.debug("No mappings to search For "); + conditionBuilder.append("FALSE"); + } + return conditionBuilder.toString(); + } + + + private String replaceWithMappings(String selectionFields) { + String toReturn=selectionFields; + // fieldLabel -> (uuid->name) + for(Entry> mapping: mappings.entrySet()) { + if(exists(mapping.getKey())) { + String actualMapped=actualField(mapping.getKey()); + if(toReturn.contains(actualMapped)) { + StringBuilder caseBuilder=new StringBuilder("CASE "+actualMapped); + for(Entry condition: mapping.getValue().entrySet()) + caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue())); + caseBuilder.append("END AS "+actualMapped); + + toReturn=toReturn.replace(actualMapped, caseBuilder.toString()); + } + } + } + + return toReturn.toString(); + } + + private String getConditionString() { ArrayList orGroups=new ArrayList(); - - + // AREA - if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && actualStructure.containsKey(schema.getAreaField())) { - String areaField=actualStructure.get(schema.getAreaField()).getFieldName(); - StringBuilder areas=new StringBuilder(); - for(String area:theRequest.getAreas()) { - areas.append(String.format("%1$s= '%2$s' OR", areaField,area)); - } - orGroups.add(areas.substring(0,areas.lastIndexOf("OR"))); - } - + if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && exists(schema.getAreaField())) + orGroups.add(getFilterByMultipleValues(theRequest.getAreas(), schema.getAreaField())); + // QUARTER - if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && actualStructure.containsKey(schema.getQuarterField())) { - String quarterField=actualStructure.get(schema.getQuarterField()).getFieldName(); - StringBuilder quarterString=new StringBuilder(); - for(String q:theRequest.getQuarters()) { - quarterString.append(String.format("%1$s= '%2$s' OR", quarterField,q)); - } - orGroups.add(quarterString.substring(0,quarterString.lastIndexOf("OR"))); - } - + if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && exists(schema.getQuarterField())) + orGroups.add(getFilterByMultipleValues(theRequest.getQuarters(), schema.getQuarterField())); + // SPECIES ID - if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && actualStructure.containsKey(schema.getSpeciesField())) { - String speciesField=actualStructure.get(schema.getSpeciesField()).getFieldName(); - StringBuilder speciesString=new StringBuilder(); - for(String s:theRequest.getSpeciesIds()) { - speciesString.append(String.format("%1$s= '%2$s' OR", speciesField,s)); - } - orGroups.add(speciesString.substring(0,speciesString.lastIndexOf("OR"))); - } - + if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && exists(schema.getSpeciesField())) + orGroups.add(getFilterByMultipleValues(theRequest.getSpeciesIds(), schema.getSpeciesField())); + + // PERIOD - if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && actualStructure.containsKey(schema.getPeriodField())) { - String periodField=actualStructure.get(schema.getPeriodField()).getFieldName(); - StringBuilder periodString=new StringBuilder(); - for(String p:theRequest.getPeriods()) { - periodString.append(String.format("%1$s= '%2$s' OR", periodField,p)); - } - orGroups.add(periodString.substring(0,periodString.lastIndexOf("OR"))); - } - + if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && exists(schema.getPeriodField())) + orGroups.add(getFilterByMultipleValues(theRequest.getPeriods(), schema.getPeriodField())); + StringBuilder toReturn=new StringBuilder(""); for(String orGroup:orGroups) { toReturn.append(orGroup+ " AND "); } if(toReturn.length()>0) - return toReturn.substring(0, toReturn.lastIndexOf("AND")); - else return toReturn.toString(); + toReturn.setLength(toReturn.lastIndexOf("AND")); + + + return toReturn.toString(); } - public void setTablename(String tablename) { - this.tablename = tablename; - } - public void setFieldList(Collection fields) { + private String getFieldList() { StringBuilder b=new StringBuilder(); - for(DBField f:fields) - b.append(f.getFieldName()+","); - fieldList=b.toString().substring(0,b.lastIndexOf(",")); + for(DBField f:actualStructure.values()) + b.append(f.getFieldName()+","); + return b.toString().substring(0,b.lastIndexOf(",")); } - @Override - public String getQuery() { - - String selectedFields=fieldList; - - for(Entry> mapping:mappings.entrySet()) { - StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey()); - for(Entry condition: mapping.getValue().entrySet()) - caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue())); - caseBuilder.append("END AS "+mapping.getKey()+","); - - selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString()); - } - - String conditionString =getConditionString(); - if(conditionString.length()>0) conditionString= "WHERE "+conditionString; - - - return String.format("SELECT %1$s FROM %2$s %3$s", - selectedFields, tablename, conditionString); + private String actualField(String label) { + return actualStructure.get(label).getFieldName(); } - - public String getQueryForFields(String...fields ) { - StringBuilder b=new StringBuilder(); - for(String f:fields) - b.append(f+","); - - String selectedFields=b.toString().substring(0,b.lastIndexOf(",")); - - for(Entry> mapping:mappings.entrySet()) { - StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey()); - for(Entry condition: mapping.getValue().entrySet()) - caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue())); - caseBuilder.append("END AS "+mapping.getKey()+","); - selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString()); - } + private boolean exists(String label) { + return actualStructure.containsKey(label); + } - String conditionString =getConditionString(); - if(conditionString.length()>0) conditionString= "WHERE "+conditionString; - - return String.format("SELECT %1$s FROM %2$s %3$s", - selectedFields, tablename, conditionString); + private String getFilterByMultipleValues(Set filterValues,String filterLabel) { + String field=actualField(filterLabel); + StringBuilder toReturn=new StringBuilder(); + for(String p:filterValues) + toReturn.append(String.format("%1%s = '%2$s' OR",field,p)); + toReturn.setLength(toReturn.lastIndexOf("OR")); + return toReturn.toString(); } } diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java b/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java index 7fec641..a7e4b56 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java +++ b/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java @@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory; public class Query { - private static final Logger log= LoggerFactory.getLogger(Query.class); + protected static final Logger log= LoggerFactory.getLogger(Query.class); protected final String query; protected final ArrayList psFields; diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java b/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java index c7f01db..f519745 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java +++ b/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java @@ -74,6 +74,12 @@ public class CSVExportRequest { periods.addAll(toAdd); return this; } + @Override + public String toString() { + return "CSVExportRequest [type=" + type + ", farmIds=" + farmIds + ", quarters=" + quarters + ", areas=" + areas + + ", speciesIds=" + speciesIds + ", periods=" + periods + "]"; + } + } diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java index 31530f3..eb6675d 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java +++ b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java @@ -19,7 +19,6 @@ import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.gcube.application.perform.service.engine.DataBaseManager; import org.gcube.application.perform.service.engine.impl.ExportCSVQuery; -import org.gcube.application.perform.service.engine.impl.ImporterImpl; import org.gcube.application.perform.service.engine.impl.Queries; import org.gcube.application.perform.service.engine.impl.Query; import org.gcube.application.perform.service.engine.impl.SchemaDefinition; @@ -36,7 +35,12 @@ import org.slf4j.LoggerFactory; public class ImportedTable { - private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class); + @Override + public String toString() { + return "ImportedTable [tablename=" + tablename + "]"; + } + + private static final Logger log= LoggerFactory.getLogger(ImportedTable.class); /** * CSV FILE labels -> DBField @@ -213,7 +217,6 @@ public class ImportedTable { try { ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels); - exportQuery.setFieldList(labels.values()); exportQuery.setTablename(tablename); Map farmMapping=new HashMap<>(); @@ -241,14 +244,22 @@ public class ImportedTable { } } - if(schema.getAssociationUUIDField()!=null) + if(schema.getAssociationUUIDField()!=null) { + log.debug("Setting Association Mapping : "+associationMapping); exportQuery.setMapping(schema.getAssociationUUIDField(), associationMapping); - if(schema.getCompanyUUIDField()!=null) + } + if(schema.getCompanyUUIDField()!=null) { + log.debug("Setting Company Mapping : "+companyMapping); exportQuery.setMapping(schema.getCompanyUUIDField(), companyMapping); - if(schema.getFarmUUIDField()!=null) + } + if(schema.getFarmUUIDField()!=null) { + log.debug("Setting Farm Mapping : "+farmMapping); exportQuery.setMapping(schema.getFarmUUIDField(), farmMapping); - if(schema.getBatchUUIDField()!=null) + } + if(schema.getBatchUUIDField()!=null) { + log.debug("Setting Batch Mapping : "+batchMapping); exportQuery.setMapping(schema.getBatchUUIDField(), batchMapping); + } log.trace("Performing actual query towards {} ",tablename); @@ -273,7 +284,12 @@ public class ImportedTable { for(String label:personalLabels) { toExtractFields.add(labels.get(label).getFieldName()); } - String sqlPersonal=exportQuery.getQueryForFields(toExtractFields.toArray(new String[toExtractFields.size()])); + + + + + String sqlPersonal=exportQuery.getQueryForMappedFields(schema.getFarmUUIDField(), + toExtractFields.toArray(new String[toExtractFields.size()])); log.debug("Query is {} ",sqlPersonal); csvRs=stmt.executeQuery(sqlPersonal); diff --git a/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java b/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java index 4cb35ac..e2ef466 100644 --- a/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java +++ b/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java @@ -3,18 +3,15 @@ package org.gcube.application.perform.service; import java.io.IOException; import java.net.MalformedURLException; import java.nio.file.Paths; -import java.sql.ResultSet; +import java.sql.Connection; import java.sql.SQLException; import org.gcube.application.perform.service.engine.DataBaseManager; import org.gcube.application.perform.service.engine.PerformanceManager; import org.gcube.application.perform.service.engine.dm.DMException; import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl; -import org.gcube.application.perform.service.engine.impl.Queries; import org.gcube.application.perform.service.engine.model.DBField; -import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; import org.gcube.application.perform.service.engine.model.InternalException; -import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor; public class LoadSchemaTest { @@ -36,13 +33,26 @@ public class LoadSchemaTest { * sourceUrl=https://data1-d.d4science.org/shub/E_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA==, sourceVersion=1.1, startTime=2019-01-25T14:52:31.442Z, endTime=null, status=ACCEPTED, lock=localhost, caller=MSgXVCkHb0SDoQLlCLQV9Kj8dPJlJ6gY+XicZJhenQkyuxA11kGUIdhxKs3jUdGK, computationId=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationUrl=http://dataminer1-pre.d4science.org:80//wps/RetrieveResultServlet?id=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationOperator=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR, computationOperatorName=Performfish Data Extractor, computationRequest=https://dataminer1-pre.d4science.org/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token=***REMOVED***&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR&DataInputs=InputData=https%3A%2F%2Fdata1-d.d4science.org%2Fshub%2FE_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA%3D%3D;BatchType=HATCHERY_INDIVIDUAL;FarmID=12682101;] */ + // RELOAD DATA FROM COMPLETED ROUTINES +// ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(), +// new DBQueryDescriptor(). +// add(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.STATUS),"COMPLETE")).executeQuery(); +// +// while(rs.next()) { +// +// ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs); +// // RELOAD DATA +// mng.loadOutputData(desc); +// // RELAUNCH REQUEST +// +// +// } - ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(), new DBQueryDescriptor(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.ID),26l)).executeQuery(); - rs.next(); - Queries.rowToDescriptor(rs); - ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs); + // CLEAN IMPORTS + Connection conn = DataBaseManager.get().getConnection(); + conn.createStatement().executeUpdate("DELETE FROM "+DBField.ImportRoutine.TABLE); + conn.commit(); - mng.loadOutputData(desc); diff --git a/src/test/java/org/gcube/application/perform/service/MiscTests.java b/src/test/java/org/gcube/application/perform/service/MiscTests.java index c4b454f..8129ec6 100644 --- a/src/test/java/org/gcube/application/perform/service/MiscTests.java +++ b/src/test/java/org/gcube/application/perform/service/MiscTests.java @@ -1,9 +1,16 @@ package org.gcube.application.perform.service; import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.core.MultivaluedHashMap; import org.gcube.application.perform.service.engine.model.DBField; import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; +import org.glassfish.jersey.internal.util.collection.ImmutableMultivaluedMap; +import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractor; +import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractorProvider; +import org.glassfish.jersey.server.model.Parameter; public class MiscTests { @@ -13,6 +20,14 @@ public class MiscTests { condition.put(DBField.Farm.fields.get(DBField.Farm.FARM_ID), "suca"); condition.put(DBField.Farm.fields.get(DBField.Farm.UUID), "boh"); System.out.println(new DBQueryDescriptor(condition)); + + + + +// theMap.put("farmid", "128"); +// ImmutableMultivaluedMap map=new ImmutableMultivaluedMap<>(theMap); +// + } } diff --git a/src/test/java/org/gcube/application/perform/service/PerformanceTest.java b/src/test/java/org/gcube/application/perform/service/PerformanceTest.java index 82559bb..bea64ba 100644 --- a/src/test/java/org/gcube/application/perform/service/PerformanceTest.java +++ b/src/test/java/org/gcube/application/perform/service/PerformanceTest.java @@ -14,11 +14,11 @@ public class PerformanceTest extends CommonTest{ public void getPerformance() { WebTarget target= target(ServiceConstants.Performance.PATH). - queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2"). - queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2"). - queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno"). - queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring"). - queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "1233556","12346"). +// queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2"). +// queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2"). +// queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno"). +// queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring"). + queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "13625424","1233556","12346"). queryParam(ServiceConstants.Performance.BATCH_TYPE_PARAMETER, "GROW_OUT_AGGREGATED"); System.out.println(target.getUri()); diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index b70dd26..5cd8cae 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -8,7 +8,9 @@ - + + + \ No newline at end of file