gCubeSystem
/
perform-service
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/application/perform-service@177079 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Fabio Sinibaldi 2019-02-12 14:35:43 +00:00
parent 1733188c79
commit 76b8d6f4ee
11 changed files with 144 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/org/gcube/application/perform/service/ServiceConstants.java
View File

@ -25,6 +25,8 @@ public interface ServiceConstants {
public static final String FARM_ID_PARAMETER="farmid";
public static final String EXCEL_FILE_PARAMETER="source";
public static final String EXCEL_FILE_VERSION_PARAMETER="source_version";
public static final String STATUS_PARAMETER="status";
public static final String LAST_METHOD="last";
}
@ -36,5 +38,8 @@ public interface ServiceConstants {
public static final String SPECIES_ID_PARAMETER="speciesid";
public static final String BATCH_TYPE_PARAMETER="batch_type";
public static final String PERIOD_PARAMETER="period";
public static final String STATISTICS_PATH="statistics";
}
}

1
src/main/java/org/gcube/application/perform/service/engine/Importer.java
View File

@ -15,4 +15,5 @@ public interface Importer {
public ImportRoutineDescriptor importExcel(ImportRequest request) throws DMException, BeanNotFound, SQLException, InternalException;
public List<ImportRoutineDescriptor> getDescriptors(DBQueryDescriptor query) throws SQLException, InternalException;
public List<ImportRoutineDescriptor> getGroupedDescriptors(DBQueryDescriptor desc) throws SQLException, InternalException ;
}

3
src/main/java/org/gcube/application/perform/service/engine/PerformanceManager.java
View File

@ -8,6 +8,7 @@ import org.gcube.application.perform.service.engine.dm.DMException;
import org.gcube.application.perform.service.engine.model.CSVExportRequest;
import org.gcube.application.perform.service.engine.model.InternalException;
import org.gcube.application.perform.service.engine.model.InvalidRequestException;
import org.gcube.application.perform.service.engine.model.importer.AnalysisType;
import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
public interface PerformanceManager {
@ -15,4 +16,6 @@ public interface PerformanceManager {
public Map<String,String> generateCSV(CSVExportRequest request)throws SQLException, InvalidRequestException, InternalException, IOException;
public void loadOutputData(ImportRoutineDescriptor desc)throws SQLException, InvalidRequestException, InternalException, IOException, DMException;
public Map<String,String> getStatistics(AnalysisType type)throws SQLException, InvalidRequestException, InternalException, IOException;
}

25
src/main/java/org/gcube/application/perform/service/engine/impl/ImporterImpl.java
View File

@ -235,8 +235,29 @@ public class ImporterImpl implements Importer {
}
/**
* Select * from imports where farmid=13625424 AND
* (batch_type,end_time) IN
* (Select batch_type,max(end_time) as end_time from imports
* WHERE farmid=13625424 AND status = 'COMPLETE' group by batch_type order by batch_type)
*
* @return
*/
@Override
public List<ImportRoutineDescriptor> getGroupedDescriptors(DBQueryDescriptor desc)throws SQLException, InternalException {
DataBaseManager db=DataBaseManager.get();
Connection conn=db.getConnection();
try {
PreparedStatement ps=Queries.LAST_GROUPED_IMPORTS.get(conn, desc);
ResultSet rs=ps.executeQuery();
ArrayList<ImportRoutineDescriptor> toReturn=new ArrayList<>();
while (rs.next())
toReturn.add(Queries.rowToDescriptor(rs));
return toReturn;
}finally {
conn.close();
}
}
}

25
src/main/java/org/gcube/application/perform/service/engine/impl/PerformanceManagerImpl.java
View File

@ -58,7 +58,7 @@ public class PerformanceManagerImpl implements PerformanceManager{
public Map<String, String> generateCSV(CSVExportRequest request) throws SQLException, InvalidRequestException, InternalException, IOException {
log.trace("Serving {} ",request);
HashMap<String,String> toReturn=new HashMap<>();
Set<ImportedTable> tables=getAnalysisSet(request);
Set<ImportedTable> tables=getAnalysisSet(request.getType());
log.debug("Found {} tables in configuration",tables.size());
for(ImportedTable t:tables) {
SchemaDefinition schema=t.getSchema();
@ -70,6 +70,24 @@ public class PerformanceManagerImpl implements PerformanceManager{
return toReturn;
}
@Override
public Map<String, String> getStatistics(AnalysisType type)
throws SQLException, InvalidRequestException, InternalException, IOException {
log.trace("Getting statistics for {} ",type);
HashMap<String,String> toReturn=new HashMap<>();
Set<ImportedTable> tables=getAnalysisSet(type);
log.debug("Found {} tables in configuration",tables.size());
for(ImportedTable t:tables) {
log.debug("Exporting {} : {} ",t.getSchema().getRelatedDescription(),t.getTableName());
toReturn.putAll(t.exportStatistics());
}
return toReturn;
}
@Override
public void loadOutputData(ImportRoutineDescriptor desc) throws SQLException, InvalidRequestException, InternalException, IOException, DMException{
log.info("Importing output for {} ",desc);
@ -135,11 +153,10 @@ public class PerformanceManagerImpl implements PerformanceManager{
static Set<ImportedTable> getAnalysisSet(CSVExportRequest request) throws InvalidRequestException{
AnalysisType type=request.getType();
static Set<ImportedTable> getAnalysisSet(AnalysisType type) throws InvalidRequestException{
if(!analysisConfiguration.containsKey(type))
throw new InvalidRequestException("Analysis Configuration not found for "+type);
return analysisConfiguration.get(request.getType());
return analysisConfiguration.get(type);
}
private static final void removeOlderEquivalents(ImportRoutineDescriptor last,Connection conn) throws SQLException, InvalidRequestException {

7
src/main/java/org/gcube/application/perform/service/engine/impl/Queries.java
View File

@ -95,6 +95,13 @@ public class Queries {
new DBField[]{DBField.ImportRoutine.fields.get(ImportRoutine.FARM_ID)});
public static final Query LAST_GROUPED_IMPORTS=new Query("Select * from "+ImportRoutine.TABLE+" where "+ImportRoutine.FARM_ID+"=? AND ("+ImportRoutine.BATCH_TYPE+","+ImportRoutine.END+") IN "
+ "(Select "+ImportRoutine.BATCH_TYPE+",max("+ImportRoutine.END+") as "+ImportRoutine.END+" from "+ImportRoutine.TABLE+" WHERE "+ImportRoutine.FARM_ID+"=? AND "+ImportRoutine.STATUS+" = ? group by "+ImportRoutine.BATCH_TYPE+")",
new DBField[]{DBField.ImportRoutine.fields.get(ImportRoutine.FARM_ID),
DBField.ImportRoutine.fields.get(ImportRoutine.FARM_ID),
DBField.ImportRoutine.fields.get(ImportRoutine.STATUS)});
public static final Query UPDATE_IMPORT_STATUS=new Query("UPDATE "+ImportRoutine.TABLE+" SET "+ImportRoutine.STATUS+"= ?, "+ImportRoutine.END+"=? WHERE "+ImportRoutine.ID+"=?",
new DBField[]{DBField.ImportRoutine.fields.get(ImportRoutine.STATUS),

43
src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java
View File

@ -208,6 +208,21 @@ public class ImportedTable {
}
public Map<String,String> exportStatistics() throws SQLException, InternalException{
log.debug("Exporting statistics from {} ",this);
Connection conn= DataBaseManager.get().getConnection();
try {
// for statistic operations
// get statistics query
// export as CSV
throw new RuntimeException("Not Yet implemented");
}finally {
conn.close();
}
}
public Map<String,String> exportCSV(CSVExportRequest request) throws InvalidRequestException, SQLException, InternalException, IOException {
@ -246,7 +261,7 @@ public class ImportedTable {
}
// Set mappings for query and csv printer
if(schema.getAssociationUUIDField()!=null) {
log.debug("Setting Association Mapping : "+associationMapping);
exportQuery.setMapping(schema.getAssociationUUIDField(), associationMapping);
@ -271,8 +286,8 @@ public class ImportedTable {
// Set mapping condition NB only farm supported at the moment
if(schema.getFarmUUIDField()!=null)
queryConverter.setCondition(schema.getFarmUUIDField(), farmMapping.keySet());
log.trace("Performing actual query towards {} ",tablename);
@ -288,31 +303,31 @@ public class ImportedTable {
toReturn.put(this.schema.getRelatedDescription(), putIntoStorage(csvRs,
csvFields.toArray(new String[csvFields.size()]),queryConverter));
if(schema.getToReportFields().size()>0) {
ArrayList<String> toExtractCSVFields=schema.getToReportFields();
queryConverter.reset();
log.trace("Extracting {} from {} ",tablename);
// Extract personal found values from same query
String[] toExtractFields=new String[toExtractCSVFields.size()];
for(String label:toExtractCSVFields) {
String fieldName=labels.get(label).getFieldName();
toExtractFields[toExtractCSVFields.indexOf(label)]=fieldName;
}
String sqlPersonal=exportQuery.getQueryForMappedFields(schema.getFarmUUIDField(),
toExtractFields);
log.debug("Query is {} ",sqlPersonal);
csvRs=stmt.executeQuery(sqlPersonal);
toReturn.put(this.schema.getRelatedDescription()+"_internal", putIntoStorage(csvRs,
schema.getToReportLabels().toArray(new String[schema.getToReportLabels().size()]),queryConverter));

28
src/main/java/org/gcube/application/perform/service/rest/Import.java
View File

@ -3,6 +3,7 @@ package org.gcube.application.perform.service.rest;
import java.util.List;
import javax.inject.Inject;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
@ -21,6 +22,7 @@ import org.gcube.application.perform.service.engine.model.DBField;
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
import org.gcube.application.perform.service.engine.model.importer.ImportRequest;
import org.gcube.application.perform.service.engine.model.importer.ImportRoutineDescriptor;
import org.gcube.application.perform.service.engine.model.importer.ImportStatus;
import org.gcube.smartgears.annotations.ManagedBy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -90,6 +92,30 @@ public class Import {
}
}
/**
* Returns all Import routine per farm id
*
*
* @return
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path(ServiceConstants.Import.LAST_METHOD+"/{"+ServiceConstants.Import.FARM_ID_PARAMETER+"}")
public Response getGrouped(@PathParam(ServiceConstants.Import.FARM_ID_PARAMETER) Long farmid,
@DefaultValue("COMPLETE") @QueryParam(ServiceConstants.Import.STATUS_PARAMETER) String status){
try {
List<ImportRoutineDescriptor> toReturn=importer.getGroupedDescriptors(
new DBQueryDescriptor().
add(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.FARM_ID), farmid).
add(DBField.ImportRoutine.fields.get(DBField.ImportRoutine.STATUS), status));
GenericEntity<List<ImportRoutineDescriptor>> entity=new GenericEntity<List<ImportRoutineDescriptor>>(toReturn) {};
return Response.ok(entity).build();
}catch(Throwable t) {
log.warn("Unexpected Exception ",t);
throw new WebApplicationException("Unexpected Exception.", t,Response.Status.INTERNAL_SERVER_ERROR);
}
}
}

8
src/main/java/org/gcube/application/perform/service/rest/Performance.java
View File

@ -7,6 +7,7 @@ import java.util.Map;
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
@ -92,6 +93,11 @@ public class Performance {
}
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path(ServiceConstants.Performance.STATISTICS_PATH+"/{"+ServiceConstants.Performance.BATCH_TYPE_PARAMETER+"}")
public Map<String,String> getStatistics(@PathParam(ServiceConstants.Performance.BATCH_TYPE_PARAMETER) String batchType){
throw new WebApplicationException("Method Unavailable.", Response.Status.NOT_IMPLEMENTED);
}
}

21
src/test/java/org/gcube/application/perform/service/ImportTests.java
View File

@ -29,11 +29,11 @@ public class ImportTests extends CommonTest{
System.out.println(target.getUri());
Response resp=target.request().post(null);
System.out.println("Status : "+resp.getStatus() );
try {
Thread.sleep(1000*60*10);
} catch (InterruptedException e) {
}
// try {
// Thread.sleep(1000*60*10);
// } catch (InterruptedException e) {
//
// }
}
@ -46,4 +46,15 @@ public class ImportTests extends CommonTest{
Response resp=target.request().get();
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
}
@Test
public void getGrouped() {
WebTarget target=
target(ServiceConstants.Import.PATH).
path(ServiceConstants.Import.LAST_METHOD).
path(13625424+"");
System.out.println(target.getUri());
Response resp=target.request().get();
System.out.println(resp.getStatus() + " : "+ resp.readEntity(String.class));
}
}

9
src/test/java/org/gcube/application/perform/service/PerformanceTest.java
View File

@ -30,15 +30,15 @@ public class PerformanceTest extends CommonTest{
File folder= Files.createTempDir();
for(AnalysisType analysis:PerformanceManagerImpl.getAnalysisConfiguration().keySet()) {
// AnalysisType analysis=new AnalysisType("GROW_OUT_INDIVIDUAL","GROW_OUT_INDIVIDUAL");
// for(AnalysisType analysis:PerformanceManagerImpl.getAnalysisConfiguration().keySet()) {
AnalysisType analysis=new AnalysisType("GROW_OUT_INDIVIDUAL","GROW_OUT_INDIVIDUAL");
WebTarget target=
target(ServiceConstants.Performance.PATH).
// queryParam(ServiceConstants.Performance.AREA_PARAMETER, "A1","A2").
//// queryParam(ServiceConstants.Performance.QUARTER_PARAMETER, "Q1","Q2").
// queryParam(ServiceConstants.Performance.SPECIES_ID_PARAMETER, "Gadilidae","Tonno").
// queryParam(ServiceConstants.Performance.PERIOD_PARAMETER, "First","Spring").
queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "13625424").
queryParam(ServiceConstants.Performance.FARM_ID_PARAMETER, "12682549").
queryParam(ServiceConstants.Performance.BATCH_TYPE_PARAMETER, analysis.getId());
System.out.println(target.getUri());
@ -58,7 +58,7 @@ public class PerformanceTest extends CommonTest{
// System.out.println(analysis.getId()+" "+resp.getStatus() + " : "+ resp.readEntity(String.class));
}
// }
System.out.println("Wrote to : "+ folder.getAbsolutePath());
}
@ -68,4 +68,5 @@ public class PerformanceTest extends CommonTest{
}