diff --git a/pom.xml b/pom.xml
index b0e3d3d..6b3ffd2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,7 +20,7 @@
${project.basedir}/src/main/webapp/WEB-INF
${project.basedir}/distro
- 2.13
+
diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java b/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java
index 99b61ae..759eb23 100644
--- a/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java
+++ b/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java
@@ -1,10 +1,10 @@
package org.gcube.application.perform.service.engine.impl;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.gcube.application.perform.service.engine.model.CSVExportRequest;
import org.gcube.application.perform.service.engine.model.DBField;
@@ -13,19 +13,20 @@ public class ExportCSVQuery extends Query {
private Map> mappings=new HashMap<>();
- private String fieldList;
private String tablename;
private CSVExportRequest theRequest;
private SchemaDefinition schema;
private Map actualStructure;
-
+
public ExportCSVQuery(String query, DBField[] fields, CSVExportRequest theRequest, SchemaDefinition schema, Map actualStructure) {
super(query, fields);
this.theRequest=theRequest;
this.schema=schema;
this.actualStructure=actualStructure;
+
+
}
@@ -34,115 +35,145 @@ public class ExportCSVQuery extends Query {
}
+
+
+ public void setTablename(String tablename) {
+ this.tablename = tablename;
+ }
+
+
+ @Override
+ public String getQuery() {
+
+ String selectedFields=replaceWithMappings(getFieldList());
+
+
+ String conditionString =getConditionString();
+ if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
+
+
+ return String.format("SELECT %1$s FROM %2$s %3$s",
+ selectedFields, tablename, conditionString);
+ }
+
+ public String getQueryForMappedFields(String filterMappingKey,String...fields) {
+ StringBuilder b=new StringBuilder();
+ for(String f:fields)
+ b.append(f+",");
+ b.setLength(b.lastIndexOf(","));
+
+ log.debug("Creating query for fields {} against table {} ",b,tablename);
+
+ String selectedFields=replaceWithMappings(b.toString());
+ String condition=getFilterWithMapping(filterMappingKey);
+
+
+
+ return String.format("SELECT %1$s FROM %2$s %3$s",
+ selectedFields, tablename, condition);
+ }
+
+ private String getFilterWithMapping(String mappingFilterKey) {
+
+ StringBuilder conditionBuilder=new StringBuilder("WHERE ");
+ if(mappings.containsKey(mappingFilterKey)&&(!mappings.get(mappingFilterKey).isEmpty())) {
+ String actualField=actualField(mappingFilterKey);
+ log.debug("Setting filter By Mappings for field {}, size {} ",actualField,mappings.get(mappingFilterKey).size());
+
+
+ for(Entry mappingFilter:mappings.get(mappingFilterKey).entrySet()) {
+ conditionBuilder.append(String.format("%1$s = '%2$s' OR", actualField,mappingFilter.getKey()));
+ }
+ conditionBuilder.setLength(conditionBuilder.lastIndexOf("OR"));
+
+ // Add selection filter..
+ String filteringCondition=getConditionString();
+ if(filteringCondition.length()>0) conditionBuilder.append(" AND "+filteringCondition);
+ }else {
+ log.debug("No mappings to search For ");
+ conditionBuilder.append("FALSE");
+ }
+ return conditionBuilder.toString();
+ }
+
+
+ private String replaceWithMappings(String selectionFields) {
+ String toReturn=selectionFields;
+ // fieldLabel -> (uuid->name)
+ for(Entry> mapping: mappings.entrySet()) {
+ if(exists(mapping.getKey())) {
+ String actualMapped=actualField(mapping.getKey());
+ if(toReturn.contains(actualMapped)) {
+ StringBuilder caseBuilder=new StringBuilder("CASE "+actualMapped);
+ for(Entry condition: mapping.getValue().entrySet())
+ caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
+ caseBuilder.append("END AS "+actualMapped);
+
+ toReturn=toReturn.replace(actualMapped, caseBuilder.toString());
+ }
+ }
+ }
+
+ return toReturn.toString();
+ }
+
+
private String getConditionString() {
ArrayList orGroups=new ArrayList();
-
-
+
// AREA
- if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && actualStructure.containsKey(schema.getAreaField())) {
- String areaField=actualStructure.get(schema.getAreaField()).getFieldName();
- StringBuilder areas=new StringBuilder();
- for(String area:theRequest.getAreas()) {
- areas.append(String.format("%1$s= '%2$s' OR", areaField,area));
- }
- orGroups.add(areas.substring(0,areas.lastIndexOf("OR")));
- }
-
+ if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && exists(schema.getAreaField()))
+ orGroups.add(getFilterByMultipleValues(theRequest.getAreas(), schema.getAreaField()));
+
// QUARTER
- if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && actualStructure.containsKey(schema.getQuarterField())) {
- String quarterField=actualStructure.get(schema.getQuarterField()).getFieldName();
- StringBuilder quarterString=new StringBuilder();
- for(String q:theRequest.getQuarters()) {
- quarterString.append(String.format("%1$s= '%2$s' OR", quarterField,q));
- }
- orGroups.add(quarterString.substring(0,quarterString.lastIndexOf("OR")));
- }
-
+ if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && exists(schema.getQuarterField()))
+ orGroups.add(getFilterByMultipleValues(theRequest.getQuarters(), schema.getQuarterField()));
+
// SPECIES ID
- if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && actualStructure.containsKey(schema.getSpeciesField())) {
- String speciesField=actualStructure.get(schema.getSpeciesField()).getFieldName();
- StringBuilder speciesString=new StringBuilder();
- for(String s:theRequest.getSpeciesIds()) {
- speciesString.append(String.format("%1$s= '%2$s' OR", speciesField,s));
- }
- orGroups.add(speciesString.substring(0,speciesString.lastIndexOf("OR")));
- }
-
+ if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && exists(schema.getSpeciesField()))
+ orGroups.add(getFilterByMultipleValues(theRequest.getSpeciesIds(), schema.getSpeciesField()));
+
+
// PERIOD
- if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && actualStructure.containsKey(schema.getPeriodField())) {
- String periodField=actualStructure.get(schema.getPeriodField()).getFieldName();
- StringBuilder periodString=new StringBuilder();
- for(String p:theRequest.getPeriods()) {
- periodString.append(String.format("%1$s= '%2$s' OR", periodField,p));
- }
- orGroups.add(periodString.substring(0,periodString.lastIndexOf("OR")));
- }
-
+ if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && exists(schema.getPeriodField()))
+ orGroups.add(getFilterByMultipleValues(theRequest.getPeriods(), schema.getPeriodField()));
+
StringBuilder toReturn=new StringBuilder("");
for(String orGroup:orGroups) {
toReturn.append(orGroup+ " AND ");
}
if(toReturn.length()>0)
- return toReturn.substring(0, toReturn.lastIndexOf("AND"));
- else return toReturn.toString();
+ toReturn.setLength(toReturn.lastIndexOf("AND"));
+
+
+ return toReturn.toString();
}
- public void setTablename(String tablename) {
- this.tablename = tablename;
- }
- public void setFieldList(Collection fields) {
+ private String getFieldList() {
StringBuilder b=new StringBuilder();
- for(DBField f:fields)
- b.append(f.getFieldName()+",");
- fieldList=b.toString().substring(0,b.lastIndexOf(","));
+ for(DBField f:actualStructure.values())
+ b.append(f.getFieldName()+",");
+ return b.toString().substring(0,b.lastIndexOf(","));
}
- @Override
- public String getQuery() {
-
- String selectedFields=fieldList;
-
- for(Entry> mapping:mappings.entrySet()) {
- StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey());
- for(Entry condition: mapping.getValue().entrySet())
- caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
- caseBuilder.append("END AS "+mapping.getKey()+",");
-
- selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString());
- }
-
- String conditionString =getConditionString();
- if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
-
-
- return String.format("SELECT %1$s FROM %2$s %3$s",
- selectedFields, tablename, conditionString);
+ private String actualField(String label) {
+ return actualStructure.get(label).getFieldName();
}
-
- public String getQueryForFields(String...fields ) {
- StringBuilder b=new StringBuilder();
- for(String f:fields)
- b.append(f+",");
-
- String selectedFields=b.toString().substring(0,b.lastIndexOf(","));
-
- for(Entry> mapping:mappings.entrySet()) {
- StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey());
- for(Entry condition: mapping.getValue().entrySet())
- caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
- caseBuilder.append("END AS "+mapping.getKey()+",");
- selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString());
- }
+ private boolean exists(String label) {
+ return actualStructure.containsKey(label);
+ }
- String conditionString =getConditionString();
- if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
-
- return String.format("SELECT %1$s FROM %2$s %3$s",
- selectedFields, tablename, conditionString);
+ private String getFilterByMultipleValues(Set filterValues,String filterLabel) {
+ String field=actualField(filterLabel);
+ StringBuilder toReturn=new StringBuilder();
+ for(String p:filterValues)
+ toReturn.append(String.format("%1%s = '%2$s' OR",field,p));
+ toReturn.setLength(toReturn.lastIndexOf("OR"));
+ return toReturn.toString();
}
}
diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java b/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java
index 7fec641..a7e4b56 100644
--- a/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java
+++ b/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java
@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
public class Query {
- private static final Logger log= LoggerFactory.getLogger(Query.class);
+ protected static final Logger log= LoggerFactory.getLogger(Query.class);
protected final String query;
protected final ArrayList psFields;
diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java b/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java
index c7f01db..f519745 100644
--- a/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java
+++ b/src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java
@@ -74,6 +74,12 @@ public class CSVExportRequest {
periods.addAll(toAdd);
return this;
}
+ @Override
+ public String toString() {
+ return "CSVExportRequest [type=" + type + ", farmIds=" + farmIds + ", quarters=" + quarters + ", areas=" + areas
+ + ", speciesIds=" + speciesIds + ", periods=" + periods + "]";
+ }
+
}
diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java
index 31530f3..eb6675d 100644
--- a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java
+++ b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java
@@ -19,7 +19,6 @@ import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.gcube.application.perform.service.engine.DataBaseManager;
import org.gcube.application.perform.service.engine.impl.ExportCSVQuery;
-import org.gcube.application.perform.service.engine.impl.ImporterImpl;
import org.gcube.application.perform.service.engine.impl.Queries;
import org.gcube.application.perform.service.engine.impl.Query;
import org.gcube.application.perform.service.engine.impl.SchemaDefinition;
@@ -36,7 +35,12 @@ import org.slf4j.LoggerFactory;
public class ImportedTable {
- private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class);
+ @Override
+ public String toString() {
+ return "ImportedTable [tablename=" + tablename + "]";
+ }
+
+ private static final Logger log= LoggerFactory.getLogger(ImportedTable.class);
/**
* CSV FILE labels -> DBField
@@ -213,7 +217,6 @@ public class ImportedTable {
try {
ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels);
- exportQuery.setFieldList(labels.values());
exportQuery.setTablename(tablename);
Map farmMapping=new HashMap<>();
@@ -241,14 +244,22 @@ public class ImportedTable {
}
}
- if(schema.getAssociationUUIDField()!=null)
+ if(schema.getAssociationUUIDField()!=null) {
+ log.debug("Setting Association Mapping : "+associationMapping);
exportQuery.setMapping(schema.getAssociationUUIDField(), associationMapping);
- if(schema.getCompanyUUIDField()!=null)
+ }
+ if(schema.getCompanyUUIDField()!=null) {
+ log.debug("Setting Company Mapping : "+companyMapping);
exportQuery.setMapping(schema.getCompanyUUIDField(), companyMapping);
- if(schema.getFarmUUIDField()!=null)
+ }
+ if(schema.getFarmUUIDField()!=null) {
+ log.debug("Setting Farm Mapping : "+farmMapping);
exportQuery.setMapping(schema.getFarmUUIDField(), farmMapping);
- if(schema.getBatchUUIDField()!=null)
+ }
+ if(schema.getBatchUUIDField()!=null) {
+ log.debug("Setting Batch Mapping : "+batchMapping);
exportQuery.setMapping(schema.getBatchUUIDField(), batchMapping);
+ }
log.trace("Performing actual query towards {} ",tablename);
@@ -273,7 +284,12 @@ public class ImportedTable {
for(String label:personalLabels) {
toExtractFields.add(labels.get(label).getFieldName());
}
- String sqlPersonal=exportQuery.getQueryForFields(toExtractFields.toArray(new String[toExtractFields.size()]));
+
+
+
+
+ String sqlPersonal=exportQuery.getQueryForMappedFields(schema.getFarmUUIDField(),
+ toExtractFields.toArray(new String[toExtractFields.size()]));
log.debug("Query is {} ",sqlPersonal);
csvRs=stmt.executeQuery(sqlPersonal);
diff --git a/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java b/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java
index 4cb35ac..e2ef466 100644
--- a/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java
+++ b/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java
@@ -3,18 +3,15 @@ package org.gcube.application.perform.service;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.file.Paths;
-import java.sql.ResultSet;
+import java.sql.Connection;
import java.sql.SQLException;
import org.gcube.application.perform.service.engine.DataBaseManager;
import org.gcube.application.perform.service.engine.PerformanceManager;
import org.gcube.application.perform.service.engine.dm.DMException;
import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl;
-import org.gcube.application.perform.service.engine.impl.Queries;
import org.gcube.application.perform.service.engine.model.DBField;
-import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
import org.gcube.application.perform.service.engine.model.InternalException;
-import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
public class LoadSchemaTest {
@@ -36,13 +33,26 @@ public class LoadSchemaTest {
* sourceUrl=https://data1-d.d4science.org/shub/E_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA==, sourceVersion=1.1, startTime=2019-01-25T14:52:31.442Z, endTime=null, status=ACCEPTED, lock=localhost, caller=MSgXVCkHb0SDoQLlCLQV9Kj8dPJlJ6gY+XicZJhenQkyuxA11kGUIdhxKs3jUdGK, computationId=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationUrl=http://dataminer1-pre.d4science.org:80//wps/RetrieveResultServlet?id=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationOperator=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR, computationOperatorName=Performfish Data Extractor, computationRequest=https://dataminer1-pre.d4science.org/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token=***REMOVED***&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR&DataInputs=InputData=https%3A%2F%2Fdata1-d.d4science.org%2Fshub%2FE_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA%3D%3D;BatchType=HATCHERY_INDIVIDUAL;FarmID=12682101;]
*/
+ // RELOAD DATA FROM COMPLETED ROUTINES
+// ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(),
+// new DBQueryDescriptor().
+// add(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.STATUS),"COMPLETE")).executeQuery();
+//
+// while(rs.next()) {
+//
+// ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs);
+// // RELOAD DATA
+// mng.loadOutputData(desc);
+// // RELAUNCH REQUEST
+//
+//
+// }
- ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(), new DBQueryDescriptor(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.ID),26l)).executeQuery();
- rs.next();
- Queries.rowToDescriptor(rs);
- ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs);
+ // CLEAN IMPORTS
+ Connection conn = DataBaseManager.get().getConnection();
+ conn.createStatement().executeUpdate("DELETE FROM "+DBField.ImportRoutine.TABLE);
+ conn.commit();
- mng.loadOutputData(desc);
diff --git a/src/test/java/org/gcube/application/perform/service/MiscTests.java b/src/test/java/org/gcube/application/perform/service/MiscTests.java
index c4b454f..8129ec6 100644
--- a/src/test/java/org/gcube/application/perform/service/MiscTests.java
+++ b/src/test/java/org/gcube/application/perform/service/MiscTests.java
@@ -1,9 +1,16 @@
package org.gcube.application.perform.service;
import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.MultivaluedHashMap;
import org.gcube.application.perform.service.engine.model.DBField;
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
+import org.glassfish.jersey.internal.util.collection.ImmutableMultivaluedMap;
+import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractor;
+import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractorProvider;
+import org.glassfish.jersey.server.model.Parameter;
public class MiscTests {
@@ -13,6 +20,14 @@ public class MiscTests {
condition.put(DBField.Farm.fields.get(DBField.Farm.FARM_ID), "suca");
condition.put(DBField.Farm.fields.get(DBField.Farm.UUID), "boh");
System.out.println(new DBQueryDescriptor(condition));
+
+
+
+
+// theMap.put("farmid", "128");
+// ImmutableMultivaluedMap map=new ImmutableMultivaluedMap<>(theMap);
+//
+
}
}
diff --git a/src/test/java/org/gcube/application/perform/service/PerformanceTest.java b/src/test/java/org/gcube/application/perform/service/PerformanceTest.java
index 82559bb..bea64ba 100644
--- a/src/test/java/org/gcube/application/perform/service/PerformanceTest.java
+++ b/src/test/java/org/gcube/application/perform/service/PerformanceTest.java
@@ -14,11 +14,11 @@ public class PerformanceTest extends CommonTest{
public void getPerformance() {
WebTarget target=
target(ServiceConstants.Performance.PATH).
- queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
- queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
- queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
- queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
- queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "1233556","12346").
+// queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
+// queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
+// queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
+// queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
+ queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "13625424","1233556","12346").
queryParam(ServiceConstants.Performance.BATCH_TYPE_PARAMETER, "GROW_OUT_AGGREGATED");
System.out.println(target.getUri());
diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml
index b70dd26..5cd8cae 100644
--- a/src/test/resources/logback.xml
+++ b/src/test/resources/logback.xml
@@ -8,7 +8,9 @@
-
+
+
+
\ No newline at end of file