From e06ab292bd15985ae00bbc00d6bd07079fd42e05 Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Wed, 23 Jan 2019 17:30:13 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/application/perform-service@176769 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../perform/service/LocalConfiguration.java | 2 ++ .../PerformServiceLifecycleManager.java | 8 +++-- .../service/PerformServiceManager.java | 3 +- .../service/engine/impl/ExportCSVQuery.java | 26 ++++++++++----- .../service/engine/impl/ImporterImpl.java | 13 ++++++-- .../engine/impl/PerformanceManagerImpl.java | 22 ++++++++++--- .../perform/service/engine/impl/Query.java | 2 +- .../perform/service/engine/model/DBField.java | 13 +++++--- .../engine/model/importer/ImportedTable.java | 33 +++++++++++-------- .../service/engine/utils/StorageUtils.java | 4 +-- src/main/webapp/WEB-INF/config.properties | 3 +- ...l_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv} | 0 ...l_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv} | 0 ...ed_Batch_Data_Entry KPI_CLOSED_BATCHES.csv | 2 -- ...egated_Batch_Data_Entry_KPI_aggregated.csv | 2 ++ ...e-grow_Batch_Data_Entry_KPI_anagraphic.csv | 2 ++ .../GROW_OUT_INDIVIDUAL/Anagraphic.properties | 3 ++ .../GROW_OUT_INDIVIDUAL/Batches.properties | 3 +- .../Anagraphic.properties | 3 ++ .../HATCHERY_AGGREGATED/Anagraphic.properties | 3 ++ .../HATCHERY_AGGREGATED/Batches.properties | 2 +- .../HATCHERY_INDIVIDUAL/Anagraphic.properties | 3 ++ .../HATCHERY_INDIVIDUAL/Batches.properties | 2 +- .../Anagraphic.properties | 3 ++ .../Batches.properties | 2 +- .../PRE_ONGROWING/Anagraphic.properties | 3 ++ .../schema/PRE_ONGROWING/Batches.properties | 6 ++-- .../Anagraphic.properties | 3 ++ .../Batches.properties | 2 +- .../perform/service/InitializeDataBase.java | 16 +++++---- .../perform/service/LoadSchemaTest.java | 6 ++-- 31 files changed, 134 insertions(+), 61 deletions(-) rename src/main/webapp/WEB-INF/csv/{Grow_out_Individual_Batch_Data_Entry KPI_CLOSED_BATCHES.csv => Grow_out_Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv} (100%) rename src/main/webapp/WEB-INF/csv/{Hatchery_ Individual_Batch_Data_Entry KPI_CLOSED_BATCHES.csv => Hatchery_ Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv} (100%) delete mode 100644 src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry KPI_CLOSED_BATCHES.csv create mode 100644 src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv create mode 100644 src/main/webapp/WEB-INF/csv/Pre-grow_Batch_Data_Entry_KPI_anagraphic.csv create mode 100644 src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Anagraphic.properties create mode 100644 src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties create mode 100644 src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Anagraphic.properties create mode 100644 src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Anagraphic.properties create mode 100644 src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties create mode 100644 src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Anagraphic.properties create mode 100644 src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Anagraphic.properties diff --git a/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java b/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java index bef5d3d..7c245f6 100644 --- a/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java +++ b/src/main/java/org/gcube/application/perform/service/LocalConfiguration.java @@ -26,6 +26,8 @@ public class LocalConfiguration { public static final String LOAD_SCHEMA="schema.load"; public static final String SKIP_ON_SCHEMA_ERROR="schema.load.skipError"; + public static final String COMMIT_SCHEMA="schema.load.commit"; + static LocalConfiguration instance=null; diff --git a/src/main/java/org/gcube/application/perform/service/PerformServiceLifecycleManager.java b/src/main/java/org/gcube/application/perform/service/PerformServiceLifecycleManager.java index 26701a6..c2cda67 100644 --- a/src/main/java/org/gcube/application/perform/service/PerformServiceLifecycleManager.java +++ b/src/main/java/org/gcube/application/perform/service/PerformServiceLifecycleManager.java @@ -33,12 +33,15 @@ public class PerformServiceLifecycleManager extends ApplicationLifecycleHandler @Override public void onStart(Start e) { + super.onStart(e); try{ - ApplicationContext context=ContextProvider.get(); + ApplicationContext context=e.context(); + log.debug("Starting Service. ApplicationContext is {} ",context); + log.debug("Application is {} ",context.application()); URL resourceUrl = context.application().getResource("/WEB-INF/config.properties"); LocalConfiguration.init(resourceUrl); - ServletContext ctx=ContextProvider.get().application(); + ServletContext ctx=context.application(); String webinfPath=ctx.getRealPath("/WEB-INF"); if(Boolean.parseBoolean(LocalConfiguration.getProperty(LocalConfiguration.LOAD_SCHEMA))) { initSchema(webinfPath); @@ -52,7 +55,6 @@ public class PerformServiceLifecycleManager extends ApplicationLifecycleHandler }catch(Exception ex) { throw new RuntimeException("Unable to init",ex); } - super.onStart(e); } diff --git a/src/main/java/org/gcube/application/perform/service/PerformServiceManager.java b/src/main/java/org/gcube/application/perform/service/PerformServiceManager.java index ac21142..fb335a4 100644 --- a/src/main/java/org/gcube/application/perform/service/PerformServiceManager.java +++ b/src/main/java/org/gcube/application/perform/service/PerformServiceManager.java @@ -3,6 +3,7 @@ package org.gcube.application.perform.service; import java.net.URL; import org.gcube.application.perform.service.engine.impl.ImporterImpl; +import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl; import org.gcube.application.perform.service.engine.utils.ScopeUtils; import org.gcube.smartgears.ApplicationManager; import org.gcube.smartgears.ContextProvider; @@ -24,7 +25,7 @@ public class PerformServiceManager implements ApplicationManager{ try { new ImporterImpl().init(); - + PerformanceManagerImpl.initDatabase(); }catch(Throwable t) { log.warn("UNABLE TO INIT SERVICE UNDER SCOPE "+ScopeUtils.getCurrentScope(), t); } diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java b/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java index dcbbe85..e30dba0 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java +++ b/src/main/java/org/gcube/application/perform/service/engine/impl/ExportCSVQuery.java @@ -19,10 +19,13 @@ public class ExportCSVQuery extends Query { private CSVExportRequest theRequest; private SchemaDefinition schema; - public ExportCSVQuery(String query, DBField[] fields, CSVExportRequest theRequest, SchemaDefinition schema) { + private Map actualStructure; + + public ExportCSVQuery(String query, DBField[] fields, CSVExportRequest theRequest, SchemaDefinition schema, Map actualStructure) { super(query, fields); this.theRequest=theRequest; this.schema=schema; + this.actualStructure=actualStructure; } @@ -35,38 +38,43 @@ public class ExportCSVQuery extends Query { ArrayList orGroups=new ArrayList(); + // AREA - if(theRequest.getAreas().size()>0) { + if(theRequest.getAreas().size()>0 && schema.getAreaField()!=null) { + String areaField=actualStructure.get(schema.getAreaField()).getFieldName(); StringBuilder areas=new StringBuilder(); for(String area:theRequest.getAreas()) { - areas.append(String.format("%1$s= '%2$s' OR", schema.getAreaField(),area)); + areas.append(String.format("%1$s= '%2$s' OR", areaField,area)); } orGroups.add(areas.substring(0,areas.lastIndexOf("OR"))); } // QUARTER - if(theRequest.getQuarters().size()>0) { + if(theRequest.getQuarters().size()>0 && schema.getQuarterField()!=null) { + String quarterField=actualStructure.get(schema.getQuarterField()).getFieldName(); StringBuilder quarterString=new StringBuilder(); for(String q:theRequest.getQuarters()) { - quarterString.append(String.format("%1$s= '%2$s' OR", schema.getQuarterField(),q)); + quarterString.append(String.format("%1$s= '%2$s' OR", quarterField,q)); } orGroups.add(quarterString.substring(0,quarterString.lastIndexOf("OR"))); } // SPECIES ID - if(theRequest.getSpeciesIds().size()>0) { + if(theRequest.getSpeciesIds().size()>0 && schema.getSpeciesField()!=null) { + String speciesField=actualStructure.get(schema.getSpeciesField()).getFieldName(); StringBuilder speciesString=new StringBuilder(); for(String s:theRequest.getSpeciesIds()) { - speciesString.append(String.format("%1$s= '%2$s' OR", schema.getSpeciesField(),s)); + speciesString.append(String.format("%1$s= '%2$s' OR", speciesField,s)); } orGroups.add(speciesString.substring(0,speciesString.lastIndexOf("OR"))); } // PERIOD - if(theRequest.getPeriods().size()>0) { + if(theRequest.getPeriods().size()>0 && schema.getPeriodField()!=null) { + String periodField=actualStructure.get(schema.getPeriodField()).getFieldName(); StringBuilder periodString=new StringBuilder(); for(String p:theRequest.getPeriods()) { - periodString.append(String.format("%1$s= '%2$s' OR", schema.getPeriodField(),p)); + periodString.append(String.format("%1$s= '%2$s' OR", periodField,p)); } orGroups.add(periodString.substring(0,periodString.lastIndexOf("OR"))); } diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/ImporterImpl.java b/src/main/java/org/gcube/application/perform/service/engine/impl/ImporterImpl.java index 83d73e3..d1aaaa6 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/impl/ImporterImpl.java +++ b/src/main/java/org/gcube/application/perform/service/engine/impl/ImporterImpl.java @@ -18,6 +18,7 @@ import org.gcube.application.perform.service.engine.dm.DMException; import org.gcube.application.perform.service.engine.dm.DMUtils; import org.gcube.application.perform.service.engine.dm.ImporterMonitor; import org.gcube.application.perform.service.engine.model.BeanNotFound; +import org.gcube.application.perform.service.engine.model.DBField; import org.gcube.application.perform.service.engine.model.DBField.ImportRoutine; import org.gcube.application.perform.service.engine.model.DBQueryDescriptor; import org.gcube.application.perform.service.engine.model.InternalException; @@ -65,18 +66,24 @@ public class ImporterImpl implements Importer { try { conn.setAutoCommit(true); - PreparedStatement psOrphans=Queries.ORPHAN_IMPORTS.prepare(conn); + String hostname=getHostname(); + + DBField lockField=ImportRoutine.fields.get(ImportRoutine.LOCK); + + PreparedStatement psOrphans=Queries.ORPHAN_IMPORTS.get(conn,new DBQueryDescriptor(lockField, hostname)); PreparedStatement psAcquire=Queries.ACQUIRE_IMPORT_ROUTINE.prepare(conn); // set ps + + + ResultSet rsOrphans=psOrphans.executeQuery(); long monitoredCount=0l; while(rsOrphans.next()) { Long id=rsOrphans.getLong(ImportRoutine.ID); try { ImportRoutineDescriptor desc=Queries.rowToDescriptor(rsOrphans); - String hostname=getHostname(); DBQueryDescriptor acquireDesc=new DBQueryDescriptor(). - add(ImportRoutine.fields.get(ImportRoutine.LOCK), hostname). + add(lockField,hostname). add(ImportRoutine.fields.get(ImportRoutine.ID), id); Queries.ACQUIRE_IMPORT_ROUTINE.fill(psAcquire, acquireDesc); diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/PerformanceManagerImpl.java b/src/main/java/org/gcube/application/perform/service/engine/impl/PerformanceManagerImpl.java index 0c62b7c..5c7d8bd 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/impl/PerformanceManagerImpl.java +++ b/src/main/java/org/gcube/application/perform/service/engine/impl/PerformanceManagerImpl.java @@ -7,6 +7,7 @@ import java.io.Reader; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Statement; import java.sql.Types; import java.util.ArrayList; import java.util.HashMap; @@ -18,6 +19,7 @@ import java.util.Set; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; +import org.gcube.application.perform.service.LocalConfiguration; import org.gcube.application.perform.service.engine.DataBaseManager; import org.gcube.application.perform.service.engine.PerformanceManager; import org.gcube.application.perform.service.engine.dm.DMUtils; @@ -83,6 +85,21 @@ public class PerformanceManagerImpl implements PerformanceManager{ } + public static void initDatabase() throws SQLException, InternalException { + DataBaseManager db=DataBaseManager.get(); + Connection conn=db.getConnection(); + Statement stmt=conn.createStatement(); + for(Entry> entry:getAnalysisConfiguration().entrySet()) { + for(ImportedTable t:entry.getValue()) { + String createStmt=t.createStatement(); + log.debug("Creating Table with stmt {} ",createStmt); + stmt.execute(createStmt); + } + } + if(Boolean.parseBoolean(LocalConfiguration.getProperty(LocalConfiguration.COMMIT_SCHEMA))) + conn.commit(); + } + public static void importSchema(SchemaDefinition schema,String csvBasePath) throws IOException, SQLException, InternalException { log.info("Loading schema {} ",schema); @@ -95,7 +112,7 @@ public class PerformanceManagerImpl implements PerformanceManager{ AnalysisType analysisType=schema.getRelatedAnalysis(); - String tablename=(analysisType.getId()+schema.getRelatedDescription()).toLowerCase().replaceAll(" ", "_"); + String tablename=(analysisType.getId()+"_"+schema.getRelatedDescription()).toLowerCase().replaceAll(" ", "_"); @@ -103,9 +120,6 @@ public class PerformanceManagerImpl implements PerformanceManager{ tablename, schema, csvFieldsDefinition); - table.create(); - - if(!analysisConfiguration.containsKey(analysisType)) analysisConfiguration.put(schema.getRelatedAnalysis(), new HashSet<>()); analysisConfiguration.get(schema.getRelatedAnalysis()).add(table); diff --git a/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java b/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java index b3bf001..7fec641 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java +++ b/src/main/java/org/gcube/application/perform/service/engine/impl/Query.java @@ -22,7 +22,7 @@ public class Query { public Query(String query,DBField[] fields) { this.query=query; - this.psFields=new ArrayList<>(Arrays.asList(fields)); + this.psFields=fields!=null?new ArrayList<>(Arrays.asList(fields)):null; } diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/DBField.java b/src/main/java/org/gcube/application/perform/service/engine/model/DBField.java index 61b3007..e9747b9 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/model/DBField.java +++ b/src/main/java/org/gcube/application/perform/service/engine/model/DBField.java @@ -113,9 +113,9 @@ public class DBField { public static final String COMPANY_UUID="companyuuid"; public static final String ASSOCIATION_UUID="associationuuid"; - public static final String FARM_LABEL="1234"; - public static final String ASSOCIATION_LABEL="1234"; - public static final String COMPANY_LABEL="1234"; + public static final String FARM_LABEL="name"; + public static final String ASSOCIATION_LABEL="association_name"; + public static final String COMPANY_LABEL="company_name"; static { @@ -123,8 +123,11 @@ public class DBField { fields.put(COMPANY_ID, new DBField(Types.BIGINT,COMPANY_ID)); fields.put(ASSOCIATION_ID, new DBField(Types.BIGINT,ASSOCIATION_ID)); fields.put(UUID, new DBField(Integer.MIN_VALUE,UUID)); - fields.put(COMPANY_UUID, new DBField(Types.VARCHAR,COMPANY_UUID)); - fields.put(ASSOCIATION_UUID, new DBField(Types.VARCHAR,ASSOCIATION_UUID)); + fields.put(COMPANY_UUID, new DBField(Integer.MIN_VALUE,COMPANY_UUID)); + fields.put(ASSOCIATION_UUID, new DBField(Integer.MIN_VALUE,ASSOCIATION_UUID)); + fields.put(FARM_LABEL, new DBField(Types.VARCHAR,FARM_LABEL)); + fields.put(ASSOCIATION_LABEL, new DBField(Types.VARCHAR,ASSOCIATION_LABEL)); + fields.put(COMPANY_LABEL, new DBField(Types.VARCHAR,COMPANY_LABEL)); } } diff --git a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java index 7da74d2..710b33b 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java +++ b/src/main/java/org/gcube/application/perform/service/engine/model/importer/ImportedTable.java @@ -68,7 +68,7 @@ public class ImportedTable { for(DBField field:csvFieldsDefinition) { String escaped=escapeString(field.getFieldName()); - csvFields.add(escaped); + csvFields.add(field.getFieldName()); labels.put(field.getFieldName(), new DBField(field.getType(),escaped)); } @@ -104,7 +104,7 @@ public class ImportedTable { return new Query(insertSQL, queryFields.toArray(new DBField[queryFields.size()])); } - public void create() throws SQLException, InternalException { + public String createStatement() { StringBuilder fieldDefinitions=new StringBuilder(); for(DBField f:labels.values()) { @@ -124,16 +124,8 @@ public class ImportedTable { String.format( "%1$s bigint," + "FOREIGN KEY (%1$s) REFERENCES "+ImportRoutine.TABLE+"("+ImportRoutine.ID+")",getRoutineIdField().getFieldName()); - String stmt=String.format("CREATE TABLE IF NOT EXISTS %1$s (%2$s, %3$s)", + return String.format("CREATE TABLE IF NOT EXISTS %1$s (%2$s, %3$s)", tablename,fieldDefinitions.substring(0,fieldDefinitions.lastIndexOf(",")),standardDefinitions); - - Connection conn=DataBaseManager.get().getConnection(); - try { - conn.createStatement().execute(stmt); - }finally { - conn.close(); - } - } @@ -187,7 +179,7 @@ public class ImportedTable { FileWriter writer=null; CSVPrinter printer=null; try { - ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema); + ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels); exportQuery.setFieldList(labels.values()); exportQuery.setTablename(tablename); @@ -231,7 +223,7 @@ public class ImportedTable { File toReturn=File.createTempFile("csv_out", ".csv"); writer=new FileWriter(toReturn); - printer = CSVFormat.DEFAULT.withHeader().print(writer); + printer = CSVFormat.DEFAULT.withHeader(csvFields.toArray(new String[csvFields.size()])).print(writer); printer.printRecords(csvRs); @@ -256,7 +248,20 @@ public class ImportedTable { return schema; } + int MAX_LENGTH=25; + private String escapeString(String fieldname) { - return fieldname; + String toReturn=fieldname; + if(toReturn.length()>MAX_LENGTH) + toReturn=toReturn.substring(0, MAX_LENGTH); + + DBField clashing=new DBField(0,"\""+toReturn+"\""); + int counter=1; + while(labels.containsValue(clashing)) { + clashing=new DBField(0,"\""+toReturn+"_"+counter+"\""); + counter++; + } + + return clashing.getFieldName(); } } diff --git a/src/main/java/org/gcube/application/perform/service/engine/utils/StorageUtils.java b/src/main/java/org/gcube/application/perform/service/engine/utils/StorageUtils.java index 7ef4652..d51536a 100644 --- a/src/main/java/org/gcube/application/perform/service/engine/utils/StorageUtils.java +++ b/src/main/java/org/gcube/application/perform/service/engine/utils/StorageUtils.java @@ -5,7 +5,6 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.util.UUID; -import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanagement.blobstorage.transport.backend.RemoteBackendException; import org.gcube.contentmanager.storageclient.wrapper.AccessType; @@ -29,7 +28,8 @@ public class StorageUtils { public static final String putOntoStorage(File source) throws RemoteBackendException, FileNotFoundException{ IClient client=getClient(); log.debug("Uploading local file "+source.getAbsolutePath()); - return client.put(true).LFile(new FileInputStream(source)).RFile(UUID.randomUUID().toString()); + String id=client.put(true).LFile(new FileInputStream(source)).RFile(UUID.randomUUID().toString()); + return client.getHttpUrl().RFile(id); } } diff --git a/src/main/webapp/WEB-INF/config.properties b/src/main/webapp/WEB-INF/config.properties index 8149292..5a33d92 100644 --- a/src/main/webapp/WEB-INF/config.properties +++ b/src/main/webapp/WEB-INF/config.properties @@ -8,6 +8,7 @@ mapping-db.ep.category=Database dm.importer.computationid=org.gcube.dataanalysis.wps.statisticalmanager.synchserver.mappedclasses.transducerers.PERFORMFISH_DATA_EXTRACTOR -schema.load=false +schema.load=true schema.load.skipError=true +schema.load.commit=true \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/csv/Grow_out_Individual_Batch_Data_Entry KPI_CLOSED_BATCHES.csv b/src/main/webapp/WEB-INF/csv/Grow_out_Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv similarity index 100% rename from src/main/webapp/WEB-INF/csv/Grow_out_Individual_Batch_Data_Entry KPI_CLOSED_BATCHES.csv rename to src/main/webapp/WEB-INF/csv/Grow_out_Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv diff --git a/src/main/webapp/WEB-INF/csv/Hatchery_ Individual_Batch_Data_Entry KPI_CLOSED_BATCHES.csv b/src/main/webapp/WEB-INF/csv/Hatchery_ Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv similarity index 100% rename from src/main/webapp/WEB-INF/csv/Hatchery_ Individual_Batch_Data_Entry KPI_CLOSED_BATCHES.csv rename to src/main/webapp/WEB-INF/csv/Hatchery_ Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv diff --git a/src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry KPI_CLOSED_BATCHES.csv b/src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry KPI_CLOSED_BATCHES.csv deleted file mode 100644 index a5d4dde..0000000 --- a/src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry KPI_CLOSED_BATCHES.csv +++ /dev/null @@ -1,2 +0,0 @@ -company_id,farm_id,operation_system,producer_association_affiliation,aggregated_batches_id,stocking_period,species,year_of_reference,number_of_batches_aggregated_in_a_single_record_n,date_of_stocking_for_the_first_batch_dd_mm_yy,larvae_at_1_dph_n,t_c_min_stocking,t_c_max_stocking,percentage_of_fish_with_non_inflated_swim_bladder_perc_on_1_dph_larvae,estimated_number_of_weaned_fish_n,rotifers_distribution_billions,artemia_distribution_kg,fish_with_head_deformities_perc_end_of_weaning,fish_with_spinal_deformities_percend_of_weaning,fish_with_fin_deformities_perc_end_of_weaning,t_c_min_end_of_weaning,t_c_max_end_of_weaning,number_of_fish_discarded_with_head_deformities_n,number_of_fish_discarded_with_spinal_deformities_n,number_of_fish_discarded_with_fin_deformities_n,fish_with_head_deformities_perc_2400_dd,fish_with_spinal_deformities_perc_2400_dd,fish_with_fin_deformities_perc_2400_dd,number_of_discarded_slow_grower_fish_n,number_of_fish_vaccinated_against_p_damselae_n,number_of_fish_vaccinated_against_v_anguillarum_n,number_of_fish_vaccinated_against_betanodavirus_n,estimated_number_of_fish_produced_n,date_of_last_closed_batch_dd_mm_yy,average_length_of_rearing_period_in_number_of_days_n,t_c_min_2400_dd,t_c_max_2400_dd,weaned_fish_perc,deformed_fish_at_2400dd_perc,discarded_slow_grower_fish_at_2400dd_perc,fish_produced_at_2400dd_perc,fish_produced_per_fte_employees_number_fish_fte_employees,survival_estimation_at_2400_dd_perc,head_deformities_at_end_of_weaning_perc,head_deformities_at_2400dd_perc,spinal_deformities_at_end_of_weaning_perc,spinal_deformities_at_2400dd_perc,fin_deformities_at_end_of_weaning_perc,fin_deformities_at_2400dd_perc,swim_bladder_non_inflation_at_500dd_perc,artemia_requirement_kg_of_artemia_million_fish_produced,rotifers_requirement_billions_rotifer_million_fish_produced,vaccinated_fish_against_p_damselae_at_2400dd_perc,vaccinated_fish_against_v_anguillarum_at_2400dd_perc,vaccinated_fish_against_betanodavirus_at_2400dd_perc,employees_2018,employees_2019,employees_2020 -Company_3b92fc3c-4df4-4cd0-8301-63502986a731,HID,Partial recirculated system,Association_6f48cb69-7859-4524-b4f9-a126b05c08f3,12.0,Natural,S.aurata,-3.0,12.0,20.0,12.0,12.0,40.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,30.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,30.0,12.0,11.0,30.0,100.0,300.0,100.0,100.0,0,100.0,12.0,12.0,12.0,12.0,12.0,12.0,12.0,1000000.0,1000000.0,100.0,100.0,100.0,12.0,12.0,0.0 diff --git a/src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv b/src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv new file mode 100644 index 0000000..c4e672f --- /dev/null +++ b/src/main/webapp/WEB-INF/csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv @@ -0,0 +1,2 @@ +Hatchery facility ID,Latitude (coordinates),Longitude (coordinates),Type of operation system,Number of FTE employees (season 2018-19) (n),Number of FTE employees (season 2019-20) (n),Number of FTE employees (season 2020-21) (n),Producer Association affiliation +12.0,12,12,Open water system,12.0,12.0,12.0,Asociación Empresarial de Acuicultura de España (APROMAR) - Spain diff --git a/src/main/webapp/WEB-INF/csv/Pre-grow_Batch_Data_Entry_KPI_anagraphic.csv b/src/main/webapp/WEB-INF/csv/Pre-grow_Batch_Data_Entry_KPI_anagraphic.csv new file mode 100644 index 0000000..15a21bb --- /dev/null +++ b/src/main/webapp/WEB-INF/csv/Pre-grow_Batch_Data_Entry_KPI_anagraphic.csv @@ -0,0 +1,2 @@ +Pre-grow facility ID,Latitude,Longitude,Type of operation system,Producer Association affiliation +12.0,12.0,12.0,Open flow-through System,Associazione Piscicoltori Italiani (API) - Italy diff --git a/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Anagraphic.properties new file mode 100644 index 0000000..b709528 --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Grow_out_Aggregated_Batch_Data_Entry_KPI_aggregated.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Batches.properties b/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Batches.properties index b17ce84..060cb73 100644 --- a/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Batches.properties +++ b/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL/Batches.properties @@ -4,4 +4,5 @@ company=company_id association=producer_association_affiliation batch=batch_id routine=internal_routine_id -csv=csv/Grow_out_Individual_Batch_Data_Entry_KPI.csv \ No newline at end of file +csv=csv/Grow_out_Individual_Batch_Data_Entry_KPI.csv +enable_analysis=true \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties new file mode 100644 index 0000000..b709528 --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/GROW_OUT_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Grow_out_Aggregated_Batch_Data_Entry_KPI_aggregated.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Anagraphic.properties new file mode 100644 index 0000000..e834baf --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Batches.properties b/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Batches.properties index 53f0c4b..052c6e7 100644 --- a/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Batches.properties +++ b/src/main/webapp/WEB-INF/schema/HATCHERY_AGGREGATED/Batches.properties @@ -2,7 +2,7 @@ description=BatchesTable farm=farm_id company=company_id association=producer_association_affiliation -batch=batch_id +batch=aggregated_batches_id routine=internal_routine_id csv=csv/Hatchery_Aggregated_Batch_Data_Entry_KPI.csv enable_analysis=true \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Anagraphic.properties new file mode 100644 index 0000000..e834baf --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Batches.properties b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Batches.properties index 9dd0c9d..2f3249b 100644 --- a/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Batches.properties +++ b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL/Batches.properties @@ -3,5 +3,5 @@ farm=farm_id company=company_id batch=batch_id routine=internal_routine_id -csv=csv/Hatchery_Individual_Batch_Data_Entry_KPI.csv +csv=csv/Hatchery_ Individual_Batch_Data_Entry_KPI.csv enable_analysis=true \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties new file mode 100644 index 0000000..e834baf --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Hatchery_Aggregated_Batch_Data_Entry_KPI_aggregated.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Batches.properties b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Batches.properties index 483c9b8..596a4d6 100644 --- a/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Batches.properties +++ b/src/main/webapp/WEB-INF/schema/HATCHERY_INDIVIDUAL_CLOSED_BATCHES/Batches.properties @@ -3,5 +3,5 @@ farm=farm_id company=company_id batch=batch_id routine=internal_routine_id -csv=csv/Hatchery_Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv +csv=csv/Hatchery_ Individual_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv enable_analysis=true \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Anagraphic.properties new file mode 100644 index 0000000..48c49a3 --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Pre-grow_Batch_Data_Entry_KPI_anagraphic.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Batches.properties b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Batches.properties index 4b23f67..2254369 100644 --- a/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Batches.properties +++ b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING/Batches.properties @@ -1,7 +1,7 @@ description=BatchesTable farm=farm_id company=company_id -association=producer_association_id -batch=aggregated_batch_id +batch=batch_id routine=internal_routine_id -csv=csv/Grow_out_Aggregated_Batch_Data_Entry_KPI.csv \ No newline at end of file +csv=csv/Pre-grow_Batch_Data_Entry_KPI.csv +enable_analysis=true \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Anagraphic.properties b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Anagraphic.properties new file mode 100644 index 0000000..48c49a3 --- /dev/null +++ b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Anagraphic.properties @@ -0,0 +1,3 @@ +description=AnagraphicTable +routine=internal_routine_id +csv=csv/Pre-grow_Batch_Data_Entry_KPI_anagraphic.csv \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Batches.properties b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Batches.properties index 2254369..5642dc1 100644 --- a/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Batches.properties +++ b/src/main/webapp/WEB-INF/schema/PRE_ONGROWING_CLOSED_BATCHES/Batches.properties @@ -3,5 +3,5 @@ farm=farm_id company=company_id batch=batch_id routine=internal_routine_id -csv=csv/Pre-grow_Batch_Data_Entry_KPI.csv +csv=csv/Pre-grow_Batch_Data_Entry_KPI_CLOSED_BATCHES.csv enable_analysis=true \ No newline at end of file diff --git a/src/test/java/org/gcube/application/perform/service/InitializeDataBase.java b/src/test/java/org/gcube/application/perform/service/InitializeDataBase.java index 5c8ed63..20b4fa9 100644 --- a/src/test/java/org/gcube/application/perform/service/InitializeDataBase.java +++ b/src/test/java/org/gcube/application/perform/service/InitializeDataBase.java @@ -58,6 +58,14 @@ public class InitializeDataBase { + "primary key ("+ImportRoutine.ID+"))"); + stmt.executeUpdate("CREATE OR REPLACE VIEW "+Farm.TABLE+" AS (" + + "Select f.farmid as "+Farm.FARM_ID+", f.uuid as "+Farm.UUID+", c.companyid as "+Farm.COMPANY_ID+", " + + "c.uuid as "+Farm.COMPANY_UUID+", a.associationid as "+Farm.ASSOCIATION_ID+", a.uuid as "+Farm.ASSOCIATION_UUID+", " + + "c.name as "+Farm.COMPANY_LABEL+", a.name as "+Farm.ASSOCIATION_LABEL+", f.name as "+Farm.FARM_LABEL+" " + + "FROM farms as f INNER JOIN companies as c ON f.companyid=c.companyid " + + "INNER JOIN associations as a ON c.associationid = a. associationid)"); + + // CREATE FARM VIEW try { ResultSet rs=stmt.executeQuery("Select * from "+Farm.TABLE); @@ -66,11 +74,7 @@ public class InitializeDataBase { } }catch(SQLException e) { // Expected error on table not found, trying to create it - stmt.executeUpdate("CREATE VIEW "+Farm.TABLE+" AS (" - + "Select f.farmid as "+Farm.FARM_ID+", f.uuid as "+Farm.UUID+", c.companyid as "+Farm.COMPANY_ID+", " - + "c.uuid as "+Farm.COMPANY_UUID+", a.associationid as "+Farm.ASSOCIATION_ID+", a.uuid as "+Farm.ASSOCIATION_UUID+" " - + "FROM farms as f INNER JOIN companies as c ON f.companyid=c.companyid " - + "INNER JOIN associations as a ON c.associationid = a. associationid)"); + } // stmt.executeQuery("CREATE VIEW suca as SELECT 1"); @@ -106,7 +110,7 @@ public class InitializeDataBase { -// conn.commit(); + conn.commit(); } diff --git a/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java b/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java index f6661d0..1473f3c 100644 --- a/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java +++ b/src/test/java/org/gcube/application/perform/service/LoadSchemaTest.java @@ -5,16 +5,18 @@ import java.net.MalformedURLException; import java.nio.file.Paths; import java.sql.SQLException; +import org.gcube.application.perform.service.engine.impl.PerformanceManagerImpl; import org.gcube.application.perform.service.engine.model.InternalException; public class LoadSchemaTest { public static void main(String[] args) throws MalformedURLException, IOException, SQLException, InternalException { - TokenSetter.set("/gcube/devsec"); LocalConfiguration.init(Paths.get("src/main/webapp/WEB-INF/config.properties").toUri().toURL()); PerformServiceLifecycleManager.initSchema("src/main/webapp/WEB-INF"); - + + TokenSetter.set("/gcube/preprod/preVRE"); + PerformanceManagerImpl.initDatabase(); } }