268 lines
7.9 KiB
Java
268 lines
7.9 KiB
Java
package org.gcube.application.perform.service.engine.model.importer;
|
|
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.sql.Connection;
|
|
import java.sql.PreparedStatement;
|
|
import java.sql.ResultSet;
|
|
import java.sql.SQLException;
|
|
import java.sql.Types;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
|
|
import org.apache.commons.csv.CSVFormat;
|
|
import org.apache.commons.csv.CSVPrinter;
|
|
import org.gcube.application.perform.service.engine.DataBaseManager;
|
|
import org.gcube.application.perform.service.engine.impl.ExportCSVQuery;
|
|
import org.gcube.application.perform.service.engine.impl.ImporterImpl;
|
|
import org.gcube.application.perform.service.engine.impl.Queries;
|
|
import org.gcube.application.perform.service.engine.impl.Query;
|
|
import org.gcube.application.perform.service.engine.impl.SchemaDefinition;
|
|
import org.gcube.application.perform.service.engine.model.CSVExportRequest;
|
|
import org.gcube.application.perform.service.engine.model.DBField;
|
|
import org.gcube.application.perform.service.engine.model.DBQueryDescriptor;
|
|
import org.gcube.application.perform.service.engine.model.InternalException;
|
|
import org.gcube.application.perform.service.engine.model.InvalidRequestException;
|
|
import org.gcube.application.perform.service.engine.model.DBField.ImportRoutine;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
public class ImportedTable {
|
|
|
|
|
|
private static final Logger log= LoggerFactory.getLogger(ImporterImpl.class);
|
|
|
|
/**
|
|
* CSV FILE labels -> DBField
|
|
*/
|
|
|
|
private Map<String,DBField> labels;
|
|
|
|
private ArrayList<String> csvFields; // Fields actually expected in csv
|
|
|
|
private String tablename;
|
|
// private DBField routineIdField;
|
|
|
|
// private String farmUUIDField;
|
|
// private String associationUUIDField;
|
|
// private String companyUUIDField;
|
|
// private String batchUUIDField;
|
|
// private Boolean analysisEnabled;
|
|
|
|
private Query insertQuery;
|
|
|
|
private SchemaDefinition schema;
|
|
|
|
|
|
public ImportedTable(String tablename, SchemaDefinition schema, ArrayList<DBField> csvFieldsDefinition) {
|
|
super();
|
|
this.schema=schema;
|
|
this.tablename=tablename;
|
|
// init
|
|
|
|
csvFields=new ArrayList<>();
|
|
labels=new HashMap<>();
|
|
|
|
for(DBField field:csvFieldsDefinition) {
|
|
String escaped=escapeString(field.getFieldName());
|
|
csvFields.add(field.getFieldName());
|
|
labels.put(field.getFieldName(), new DBField(field.getType(),escaped));
|
|
}
|
|
|
|
insertQuery=prepareInsertionQuery();
|
|
|
|
}
|
|
|
|
private DBField getRoutineIdField() {
|
|
return new DBField(Types.BIGINT,schema.getRoutineIdFieldName());
|
|
}
|
|
|
|
|
|
private Query prepareInsertionQuery() {
|
|
|
|
StringBuilder fieldList=new StringBuilder();
|
|
StringBuilder valueString=new StringBuilder();
|
|
ArrayList<DBField> queryFields=new ArrayList<>();
|
|
|
|
|
|
for(DBField f:labels.values()) {
|
|
queryFields.add(f);
|
|
fieldList.append(f.getFieldName()+",");
|
|
valueString.append("?,");
|
|
}
|
|
|
|
queryFields.add(getRoutineIdField());
|
|
|
|
|
|
String insertSQL= String.format("INSERT INTO %1$s (%2$s) VALUES (%3$s)", tablename,
|
|
fieldList+getRoutineIdField().getFieldName(),valueString+"?");
|
|
|
|
|
|
return new Query(insertSQL, queryFields.toArray(new DBField[queryFields.size()]));
|
|
}
|
|
|
|
public String createStatement() {
|
|
StringBuilder fieldDefinitions=new StringBuilder();
|
|
|
|
for(DBField f:labels.values()) {
|
|
String type="text";
|
|
switch(f.getType()) {
|
|
case Types.BIGINT : type="bigint";
|
|
break;
|
|
case Types.REAL : type="real";
|
|
break;
|
|
}
|
|
|
|
fieldDefinitions.append(f.getFieldName()+" "+type+",");
|
|
|
|
}
|
|
|
|
String standardDefinitions=
|
|
String.format( "%1$s bigint,"
|
|
+ "FOREIGN KEY (%1$s) REFERENCES "+ImportRoutine.TABLE+"("+ImportRoutine.ID+")",getRoutineIdField().getFieldName());
|
|
|
|
return String.format("CREATE TABLE IF NOT EXISTS %1$s (%2$s, %3$s)",
|
|
tablename,fieldDefinitions.substring(0,fieldDefinitions.lastIndexOf(",")),standardDefinitions);
|
|
}
|
|
|
|
|
|
/**
|
|
* Checks if passed set of labels is
|
|
*
|
|
* @param toMatchSchema
|
|
* @return
|
|
*/
|
|
public boolean matchesSchema(ArrayList<String> toMatchSchema) {
|
|
return csvFields.equals(toMatchSchema);
|
|
}
|
|
|
|
|
|
public DBQueryDescriptor getSetRow(Map<String,String> csvRow, Long routineId) {
|
|
DBQueryDescriptor desc=new DBQueryDescriptor();
|
|
|
|
for(Entry<String,String> csvField:csvRow.entrySet()) {
|
|
DBField toSetField=labels.get(csvField.getKey());
|
|
|
|
Object value=csvField.getValue();
|
|
if(value!=null)
|
|
switch(toSetField.getType()) {
|
|
case Types.BIGINT : value=Long.parseLong((String) value);
|
|
break;
|
|
case Types.REAL : value=Double.parseDouble((String) value);
|
|
break;
|
|
}
|
|
desc.add(toSetField, value);
|
|
}
|
|
|
|
desc.add(getRoutineIdField(), routineId);
|
|
return desc;
|
|
}
|
|
|
|
public Query getInsertQuery() {
|
|
return insertQuery;
|
|
}
|
|
public String getTableName() {
|
|
return tablename;
|
|
}
|
|
|
|
|
|
|
|
public File exportCSV(CSVExportRequest request) throws InvalidRequestException, SQLException, InternalException, IOException {
|
|
|
|
|
|
log.debug("Exporting {} from {} ",request, this);
|
|
|
|
Connection conn= DataBaseManager.get().getConnection();
|
|
FileWriter writer=null;
|
|
CSVPrinter printer=null;
|
|
try {
|
|
ExportCSVQuery exportQuery=new ExportCSVQuery("",null,request,schema,labels);
|
|
exportQuery.setFieldList(labels.values());
|
|
exportQuery.setTablename(tablename);
|
|
|
|
Map<String,String> farmMapping=new HashMap<>();
|
|
Map<String,String> companyMapping=new HashMap<>();
|
|
Map<String,String> associationMapping=new HashMap<>();
|
|
Map<String,String> batchMapping=new HashMap<>();
|
|
|
|
PreparedStatement psFarm=Queries.GET_FARM_BY_ID.prepare(conn);
|
|
PreparedStatement psBatch=Queries.GET_BATCH_BY_FARM_ID.prepare(conn);
|
|
|
|
|
|
// GET Labels by Farm Id
|
|
for(Long farmid:request.getFarmIds()) {
|
|
ResultSet rsFarm=Queries.GET_FARM_BY_ID.fill(psFarm, new DBQueryDescriptor(DBField.Farm.fields.get(DBField.Farm.FARM_ID),farmid)).executeQuery();
|
|
if(! rsFarm.next())
|
|
log.warn("Unable to Find farmID "+farmid);
|
|
else {
|
|
farmMapping.put(rsFarm.getString(DBField.Farm.UUID), rsFarm.getString(DBField.Farm.FARM_LABEL));
|
|
companyMapping.put(rsFarm.getString(DBField.Farm.COMPANY_UUID),rsFarm.getString(DBField.Farm.COMPANY_LABEL));
|
|
associationMapping.put(rsFarm.getString(DBField.Farm.ASSOCIATION_UUID), rsFarm.getString(DBField.Farm.ASSOCIATION_LABEL));
|
|
ResultSet rsBatch=Queries.GET_BATCH_BY_FARM_ID.fill(psBatch, new DBQueryDescriptor(DBField.Batch.fields.get(DBField.Batch.FARM_ID),farmid)).executeQuery();
|
|
while(rsBatch.next())
|
|
batchMapping.put(rsBatch.getString(DBField.Batch.UUID), rsBatch.getString(DBField.Batch.BATCH_NAME));
|
|
|
|
}
|
|
}
|
|
|
|
exportQuery.setMapping(schema.getAssociationUUIDField(), associationMapping);
|
|
exportQuery.setMapping(schema.getCompanyUUIDField(), companyMapping);
|
|
exportQuery.setMapping(schema.getFarmUUIDField(), farmMapping);
|
|
exportQuery.setMapping(schema.getBatchUUIDField(), batchMapping);
|
|
|
|
|
|
log.trace("Performing actual query towards {} ",tablename);
|
|
|
|
String sqlExport=exportQuery.getQuery();
|
|
log.debug("Query is {} ",sqlExport);
|
|
ResultSet csvRs=conn.createStatement().executeQuery(sqlExport);
|
|
|
|
File toReturn=File.createTempFile("csv_out", ".csv");
|
|
writer=new FileWriter(toReturn);
|
|
|
|
printer = CSVFormat.DEFAULT.withHeader(csvFields.toArray(new String[csvFields.size()])).print(writer);
|
|
|
|
printer.printRecords(csvRs);
|
|
|
|
|
|
return toReturn;
|
|
}finally {
|
|
conn.close();
|
|
if(writer!=null) {
|
|
writer.flush();
|
|
writer.close();
|
|
}
|
|
if(printer!=null) {
|
|
printer.flush();
|
|
printer.close();
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
|
|
public SchemaDefinition getSchema() {
|
|
return schema;
|
|
}
|
|
|
|
int MAX_LENGTH=25;
|
|
|
|
private String escapeString(String fieldname) {
|
|
String toReturn=fieldname;
|
|
if(toReturn.length()>MAX_LENGTH)
|
|
toReturn=toReturn.substring(0, MAX_LENGTH);
|
|
|
|
DBField clashing=new DBField(0,"\""+toReturn+"\"");
|
|
int counter=1;
|
|
while(labels.containsValue(clashing)) {
|
|
clashing=new DBField(0,"\""+toReturn+"_"+counter+"\"");
|
|
counter++;
|
|
}
|
|
|
|
return clashing.getFieldName();
|
|
}
|
|
}
|