git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/application/perform-service@176944 82a268e6-3cf1-43bd-a215-b396298e98cf

pom.xml

@@ -20,7 +20,7 @@
 	<properties>
 		<webappDirectory>${project.basedir}/src/main/webapp/WEB-INF</webappDirectory>
 		<distroDirectory>${project.basedir}/distro</distroDirectory>
-		<jersey-version>2.13</jersey-version>
+		<!-- <jersey-version>2.13</jersey-version> -->
 	</properties>
 
 	<dependencyManagement>

src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java

@@ -1,10 +1,10 @@
 package org.gcube.application.perform.service.engine.impl;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.gcube.application.perform.service.engine.model.CSVExportRequest;
 import org.gcube.application.perform.service.engine.model.DBField;
@@ -13,19 +13,20 @@ public class ExportCSVQuery extends Query {
 
 	private Map<String,Map<String,String>> mappings=new HashMap<>(); 
 
-	private String fieldList;
 	private String tablename;
 
 	private CSVExportRequest theRequest;
 	private SchemaDefinition schema;
 
 	private Map<String,DBField> actualStructure;
-	
+
 	public ExportCSVQuery(String query, DBField[] fields, CSVExportRequest theRequest, SchemaDefinition schema, Map<String,DBField> actualStructure) {
 		super(query, fields);
 		this.theRequest=theRequest;
 		this.schema=schema;
 		this.actualStructure=actualStructure;
+
+
 	}
 
 
@@ -34,115 +35,145 @@ public class ExportCSVQuery extends Query {
 	}
 
 
+
+
+	public void setTablename(String tablename) {
+		this.tablename = tablename;
+	}
+
+
+	@Override
+	public String getQuery() {
+
+		String selectedFields=replaceWithMappings(getFieldList());
+
+
+		String conditionString =getConditionString();
+		if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
+
+
+		return String.format("SELECT %1$s FROM %2$s %3$s", 
+				selectedFields, tablename, conditionString);
+	}
+
+	public String getQueryForMappedFields(String filterMappingKey,String...fields) {
+		StringBuilder b=new StringBuilder();
+		for(String f:fields) 
+			b.append(f+",");
+		b.setLength(b.lastIndexOf(","));
+
+		log.debug("Creating query for fields {} against table {} ",b,tablename);
+
+		String selectedFields=replaceWithMappings(b.toString());
+		String condition=getFilterWithMapping(filterMappingKey);
+
+
+
+		return String.format("SELECT %1$s FROM %2$s %3$s", 
+				selectedFields, tablename, condition);
+	}
+
+	private String getFilterWithMapping(String mappingFilterKey) {
+
+		StringBuilder conditionBuilder=new StringBuilder("WHERE ");
+		if(mappings.containsKey(mappingFilterKey)&&(!mappings.get(mappingFilterKey).isEmpty())) {
+			String actualField=actualField(mappingFilterKey);
+			log.debug("Setting filter By Mappings for field {}, size {} ",actualField,mappings.get(mappingFilterKey).size());
+
+
+			for(Entry<String,String> mappingFilter:mappings.get(mappingFilterKey).entrySet()) {
+				conditionBuilder.append(String.format("%1$s = '%2$s' OR", actualField,mappingFilter.getKey()));
+			}
+			conditionBuilder.setLength(conditionBuilder.lastIndexOf("OR"));
+			
+			// Add selection filter.. 
+			String filteringCondition=getConditionString();
+			if(filteringCondition.length()>0) conditionBuilder.append(" AND "+filteringCondition);			
+		}else {
+			log.debug("No mappings to search For ");
+			conditionBuilder.append("FALSE"); 
+		}
+		return conditionBuilder.toString();
+	}
+
+
+	private String replaceWithMappings(String selectionFields) {
+		String toReturn=selectionFields;
+		// fieldLabel -> (uuid->name)
+		for(Entry<String,Map<String,String>> mapping: mappings.entrySet()) {
+			if(exists(mapping.getKey())) {
+				String actualMapped=actualField(mapping.getKey());
+				if(toReturn.contains(actualMapped)) {
+					StringBuilder caseBuilder=new StringBuilder("CASE "+actualMapped);
+					for(Entry<String,String> condition: mapping.getValue().entrySet())
+						caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
+					caseBuilder.append("END AS "+actualMapped);
+
+					toReturn=toReturn.replace(actualMapped, caseBuilder.toString());
+				}
+			}
+		}
+
+		return toReturn.toString();		
+	}
+
+
 	private String getConditionString() {
 
 		ArrayList<String> orGroups=new ArrayList<String>();
-		
+
-		
 		// AREA
-		if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && actualStructure.containsKey(schema.getAreaField())) {
+		if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null && exists(schema.getAreaField())) 
-			String areaField=actualStructure.get(schema.getAreaField()).getFieldName();
+			orGroups.add(getFilterByMultipleValues(theRequest.getAreas(), schema.getAreaField()));
-			StringBuilder areas=new StringBuilder();		
+
-			for(String area:theRequest.getAreas()) {
-				areas.append(String.format("%1$s= '%2$s' OR", areaField,area));
-			}
-			orGroups.add(areas.substring(0,areas.lastIndexOf("OR")));
-		}
-		
 		// QUARTER
-		if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && actualStructure.containsKey(schema.getQuarterField())) {
+		if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null && exists(schema.getQuarterField())) 
-			String quarterField=actualStructure.get(schema.getQuarterField()).getFieldName();
+			orGroups.add(getFilterByMultipleValues(theRequest.getQuarters(), schema.getQuarterField()));
-			StringBuilder quarterString=new StringBuilder();		
+
-			for(String q:theRequest.getQuarters()) {
-				quarterString.append(String.format("%1$s= '%2$s' OR", quarterField,q));
-			}
-			orGroups.add(quarterString.substring(0,quarterString.lastIndexOf("OR")));
-		}
-		
 		// SPECIES ID 
-		if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && actualStructure.containsKey(schema.getSpeciesField())) {
+		if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null && exists(schema.getSpeciesField())) 
-			String speciesField=actualStructure.get(schema.getSpeciesField()).getFieldName();
+			orGroups.add(getFilterByMultipleValues(theRequest.getSpeciesIds(), schema.getSpeciesField()));
-			StringBuilder speciesString=new StringBuilder();		
+
-			for(String s:theRequest.getSpeciesIds()) {
+
-				speciesString.append(String.format("%1$s= '%2$s' OR", speciesField,s));
-			}
-			orGroups.add(speciesString.substring(0,speciesString.lastIndexOf("OR")));
-		}
-		
 		// PERIOD
-		if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && actualStructure.containsKey(schema.getPeriodField())) {
+		if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null && exists(schema.getPeriodField())) 
-			String periodField=actualStructure.get(schema.getPeriodField()).getFieldName();
+			orGroups.add(getFilterByMultipleValues(theRequest.getPeriods(), schema.getPeriodField()));
-			StringBuilder periodString=new StringBuilder();		
+
-			for(String p:theRequest.getPeriods()) {
-				periodString.append(String.format("%1$s= '%2$s' OR", periodField,p));
-			}
-			orGroups.add(periodString.substring(0,periodString.lastIndexOf("OR")));
-		}
-		
 		StringBuilder toReturn=new StringBuilder("");
 		for(String orGroup:orGroups) {
 			toReturn.append(orGroup+ " AND ");
 		}
 
 		if(toReturn.length()>0)
-		return toReturn.substring(0, toReturn.lastIndexOf("AND"));
+			toReturn.setLength(toReturn.lastIndexOf("AND"));
-		else return toReturn.toString();
+
+
+		return toReturn.toString();
 
 	}
 
-	public void setTablename(String tablename) {
-		this.tablename = tablename;
-	}
 
-	public void setFieldList(Collection<DBField> fields) {
+	private String getFieldList() {
 		StringBuilder b=new StringBuilder();
-		for(DBField f:fields) 
+		for(DBField f:actualStructure.values()) 
-			b.append(f.getFieldName()+",");
+			b.append(f.getFieldName()+",");		
-		fieldList=b.toString().substring(0,b.lastIndexOf(","));
+		return b.toString().substring(0,b.lastIndexOf(","));
 	}
 
-	@Override
+	private String actualField(String label) {
-	public String getQuery() {
+		return actualStructure.get(label).getFieldName();
-
-		String selectedFields=fieldList;
-
-		for(Entry<String,Map<String,String>> mapping:mappings.entrySet()) {
-			StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey());
-			for(Entry<String,String> condition: mapping.getValue().entrySet())
-				caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
-			caseBuilder.append("END AS "+mapping.getKey()+",");
-
-			selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString());
-		}
-
-		String conditionString =getConditionString();
-		if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
-		
-
-		return String.format("SELECT %1$s FROM %2$s %3$s", 
-				selectedFields, tablename, conditionString);
 	}
-	
-	public String getQueryForFields(String...fields ) {
-		StringBuilder b=new StringBuilder();
-		for(String f:fields) 
-			b.append(f+",");
-		
-		String selectedFields=b.toString().substring(0,b.lastIndexOf(","));
-		
-		for(Entry<String,Map<String,String>> mapping:mappings.entrySet()) {
-			StringBuilder caseBuilder=new StringBuilder("CASE "+mapping.getKey());
-			for(Entry<String,String> condition: mapping.getValue().entrySet())
-				caseBuilder.append(String.format("WHEN '%1$s' THEN '%2$s'", condition.getKey(),condition.getValue()));
-			caseBuilder.append("END AS "+mapping.getKey()+",");
 
-			selectedFields.replaceAll(mapping.getKey(), caseBuilder.toString());
+	private boolean exists(String label) {
-		}
+		return actualStructure.containsKey(label);
+	}
 
-		String conditionString =getConditionString();
+	private String getFilterByMultipleValues(Set<String> filterValues,String filterLabel) {
-		if(conditionString.length()>0) conditionString= "WHERE "+conditionString;
+		String field=actualField(filterLabel);
-		
+		StringBuilder toReturn=new StringBuilder();
-		return String.format("SELECT %1$s FROM %2$s %3$s", 
+		for(String p:filterValues)
-				selectedFields, tablename, conditionString);
+			toReturn.append(String.format("%1%s = '%2$s' OR",field,p));
+		toReturn.setLength(toReturn.lastIndexOf("OR"));
+		return toReturn.toString();
 	}
 }

src/main/java/org/gcube/application/perform/service/engine/impl/Query.java

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
 
 public class Query {
 
-	private static final Logger log= LoggerFactory.getLogger(Query.class);
+	protected static final Logger log= LoggerFactory.getLogger(Query.class);
 
 	protected final String query;
 	protected final ArrayList<DBField> psFields;

src/main/java/org/gcube/application/perform/service/engine/model/CSVExportRequest.java

@@ -74,6 +74,12 @@ public class CSVExportRequest {
 		periods.addAll(toAdd);
 		return this;
 	}
+	@Override
+	public String toString() {
+		return "CSVExportRequest [type=" + type + ", farmIds=" + farmIds + ", quarters=" + quarters + ", areas=" + areas
+				+ ", speciesIds=" + speciesIds + ", periods=" + periods + "]";
+	}
+	
 	
 	
 }

src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java

@@ -19,7 +19,6 @@ import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
 import org.gcube.application.perform.service.engine.DataBaseManager;
 import org.gcube.application.perform.service.engine.impl.ExportCSVQuery;
-import org.gcube.application.perform.service.engine.impl.ImporterImpl;
 import org.gcube.application.perform.service.engine.impl.Queries;
 import org.gcube.application.perform.service.engine.impl.Query;
 import org.gcube.application.perform.service.engine.impl.SchemaDefinition;
@@ -36,7 +35,12 @@ import org.slf4j.LoggerFactory;
 public class ImportedTable {
 
 
-	private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class);
+	@Override
+	public String toString() {
+		return "ImportedTable [tablename=" + tablename + "]";
+	}
+
+	private static final Logger log= LoggerFactory.getLogger(ImportedTable.class);
 
 	/**
 	 * CSV FILE labels -> DBField 
@@ -213,7 +217,6 @@ public class ImportedTable {
 
 		try {
 			ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels);
-			exportQuery.setFieldList(labels.values());
 			exportQuery.setTablename(tablename);
 
 			Map<String,String> farmMapping=new HashMap<>();
@@ -241,14 +244,22 @@ public class ImportedTable {
 				}
 			}
 
-			if(schema.getAssociationUUIDField()!=null)
+			if(schema.getAssociationUUIDField()!=null) {
+				log.debug("Setting Association Mapping : "+associationMapping);
 				exportQuery.setMapping(schema.getAssociationUUIDField(), associationMapping);
-			if(schema.getCompanyUUIDField()!=null)
+			}
+			if(schema.getCompanyUUIDField()!=null) {
+				log.debug("Setting Company Mapping : "+companyMapping);
 				exportQuery.setMapping(schema.getCompanyUUIDField(), companyMapping);
-			if(schema.getFarmUUIDField()!=null)
+			}
+			if(schema.getFarmUUIDField()!=null) {
+				log.debug("Setting Farm Mapping : "+farmMapping);
 				exportQuery.setMapping(schema.getFarmUUIDField(), farmMapping);
-			if(schema.getBatchUUIDField()!=null)
+			}
+			if(schema.getBatchUUIDField()!=null) {
+				log.debug("Setting Batch Mapping : "+batchMapping);
 				exportQuery.setMapping(schema.getBatchUUIDField(), batchMapping);
+			}
 
 
 			log.trace("Performing actual query towards {} ",tablename);
@@ -273,7 +284,12 @@ public class ImportedTable {
 				for(String label:personalLabels) {
 					toExtractFields.add(labels.get(label).getFieldName());
 				}
-				String sqlPersonal=exportQuery.getQueryForFields(toExtractFields.toArray(new String[toExtractFields.size()]));
+				
+				
+				
+				
+				String sqlPersonal=exportQuery.getQueryForMappedFields(schema.getFarmUUIDField(),
+						toExtractFields.toArray(new String[toExtractFields.size()]));
 				log.debug("Query is {} ",sqlPersonal);
 				csvRs=stmt.executeQuery(sqlPersonal);
 

src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java

@@ -3,18 +3,15 @@ package org.gcube.application.perform.service;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.nio.file.Paths;
-import java.sql.ResultSet;
+import java.sql.Connection;
 import java.sql.SQLException;
 
 import org.gcube.application.perform.service.engine.DataBaseManager;
 import org.gcube.application.perform.service.engine.PerformanceManager;
 import org.gcube.application.perform.service.engine.dm.DMException;
 import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl;
-import org.gcube.application.perform.service.engine.impl.Queries;
 import org.gcube.application.perform.service.engine.model.DBField;
-import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
 import org.gcube.application.perform.service.engine.model.InternalException;
-import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
 
 public class LoadSchemaTest {
 
@@ -36,13 +33,26 @@ public class LoadSchemaTest {
 		 * sourceUrl=https://data1-d.d4science.org/shub/E_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA==, sourceVersion=1.1, startTime=2019-01-25T14:52:31.442Z, endTime=null, status=ACCEPTED, lock=localhost, caller=MSgXVCkHb0SDoQLlCLQV9Kj8dPJlJ6gY+XicZJhenQkyuxA11kGUIdhxKs3jUdGK, computationId=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationUrl=http://dataminer1-pre.d4science.org:80//wps/RetrieveResultServlet?id=7c6f5d7e-b778-42be-af20-b99229fa99ee, computationOperator=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR, computationOperatorName=Performfish Data Extractor, computationRequest=https://dataminer1-pre.d4science.org/wps/WebProcessingService?request=Execute&service=WPS&Version=1.0.0&gcube-token=***REMOVED***&lang=en-US&Identifier=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR&DataInputs=InputData=https%3A%2F%2Fdata1-d.d4science.org%2Fshub%2FE_aUJUbDNzeUlLL29KL2xlZUloWFQ5TEdlZ0ZnZzlNNTVLNUEzeDVRNFVoVHlLMW5DVG5RbVVXVzlYeUUzZWFXRA%3D%3D;BatchType=HATCHERY_INDIVIDUAL;FarmID=12682101;]
 		 */
 
+		// RELOAD DATA FROM COMPLETED ROUTINES
+//		ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(), 
+//				new DBQueryDescriptor().
+//				add(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.STATUS),"COMPLETE")).executeQuery();
+//				
+//		while(rs.next()) {
+//		
+//		ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs);
+//		// RELOAD DATA 
+//			mng.loadOutputData(desc);
+//		// RELAUNCH REQUEST
+//			
+//			
+//		}
 		
-		ResultSet rs=Queries.GET_IMPORT_ROUTINE_BY_ID.get(DataBaseManager.get().getConnection(), new DBQueryDescriptor(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.ID),26l)).executeQuery();
+		// CLEAN IMPORTS 
-		rs.next();
+		Connection conn = DataBaseManager.get().getConnection();
-		Queries.rowToDescriptor(rs);
+		conn.createStatement().executeUpdate("DELETE FROM "+DBField.ImportRoutine.TABLE);
-		ImportRoutineDescriptor desc=Queries.rowToDescriptor(rs);
+		conn.commit();
 		
-		mng.loadOutputData(desc);
 		
 		
 		

src/test/java/org/gcube/application/perform/service/MiscTests.java

@@ -1,9 +1,16 @@
 package org.gcube.application.perform.service;
 
 import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.core.MultivaluedHashMap;
 
 import org.gcube.application.perform.service.engine.model.DBField;
 import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
+import org.glassfish.jersey.internal.util.collection.ImmutableMultivaluedMap;
+import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractor;
+import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractorProvider;
+import org.glassfish.jersey.server.model.Parameter;
 
 public class MiscTests {
 
@@ -13,6 +20,14 @@ public class MiscTests {
 		condition.put(DBField.Farm.fields.get(DBField.Farm.FARM_ID), "suca");
 		condition.put(DBField.Farm.fields.get(DBField.Farm.UUID), "boh");
 		System.out.println(new DBQueryDescriptor(condition));
+		
+		
+		
+	
+//		theMap.put("farmid", "128");
+//		ImmutableMultivaluedMap<String, String> map=new ImmutableMultivaluedMap<>(theMap);
+//		
+		
 	}
 
 }

src/test/java/org/gcube/application/perform/service/PerformanceTest.java

@@ -14,11 +14,11 @@ public class PerformanceTest extends CommonTest{
 	public void getPerformance() {
 		WebTarget target=
 				target(ServiceConstants.Performance.PATH).
-				queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
+//				queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
-				queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
+//				queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
-				queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
+//				queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
-				queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
+//				queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
-				queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "1233556","12346").
+				queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "13625424","1233556","12346").
 				queryParam(ServiceConstants.Performance.BATCH_TYPE_PARAMETER, "GROW_OUT_AGGREGATED");
 
 		System.out.println(target.getUri());		

src/test/resources/logback.xml

@@ -8,7 +8,9 @@
     </encoder>
   </appender>
 
-  <root level="debug">
+	<logger name="org.gcube.application" level="DEBUG"/>
+
+  <root level="WARN">
     <appender-ref ref="STDOUT" />
   </root>
 </configuration>