From 72b8e87e81fc6d721122c9f32970f98b0cf3086a Mon Sep 17 00:00:00 2001 From: Loredana Liccardo Date: Wed, 19 Nov 2014 14:38:18 +0000 Subject: [PATCH] - method submitQuery modified in class DatabaseManagement in order to return a file and a map as output. The map is truncated to 1000 rows if the result has more than 1000 rows. method writeTableIntoFile reintroduced that performs the file writing and the map generation. - Version in pom file changed in 1.3.0 - method submitQuery modified in class DatabaseManagement adding the UUID string to the submitQuery result file QueryResult file. git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-access/DatabasesResourcesManager@101653 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../databases/utils/DatabaseManagement.java | 488 ++++++++++-------- 1 file changed, 284 insertions(+), 204 deletions(-) diff --git a/src/main/java/org/gcube/dataaccess/databases/utils/DatabaseManagement.java b/src/main/java/org/gcube/dataaccess/databases/utils/DatabaseManagement.java index 4304a6a..f1675b4 100644 --- a/src/main/java/org/gcube/dataaccess/databases/utils/DatabaseManagement.java +++ b/src/main/java/org/gcube/dataaccess/databases/utils/DatabaseManagement.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.UUID; import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; import org.gcube.dataaccess.databases.converter.SqlDialectConverter; @@ -344,12 +345,12 @@ public class DatabaseManagement { */ // method that allows to submit a query - public List submitQuery(String query, Connection conn) throws Exception { + public List submitQuery(String query, Connection conn, String pathFile) throws Exception { List results = new ArrayList(); results = connection.executeQueryJDBC(query, conn); - + if (results != null) { AnalysisLogger.getLogger().debug( "DatabaseManagement-> Query result retrieved"); @@ -365,22 +366,66 @@ public class DatabaseManagement { // to retrieve datatype columns of a table List DataTypeColumns = null; - // // store table in a file - // String FileName = pathFile + "QueryResult.csv"; - // // write the result in the file and in the map - // AnalysisLogger.getLogger().debug( - // "In DatabaseManagement->writing the result in the file: " - // + FileName); - // file = new File(FileName); - // out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream( - // file), "UTF-8")); - // writeTableIntoFile(results, DataTypeColumns); + // store table in a file and build the result Map + String FileName = pathFile + "QueryResult"+UUID.randomUUID()+".csv"; + // write the result in the file and in the map + AnalysisLogger.getLogger().debug( + "In DatabaseManagement-> writing the result in the file: " + + FileName); + + try{ + file = new File(FileName); + out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + file), "UTF-8")); + writeTableIntoFile(results, DataTypeColumns); + +// //truncate the result list to 10000 rows +// if ((mapResult!=null)&&(mapResult.size()!=0)){ +// //result size without header +// int mapSize = mapResult.size() - 1; +// AnalysisLogger.getLogger().debug( +// "DatabaseManagement-> size of the map without header obtained from the submit operation: " +// + mapSize); +// if (mapSize>10000){ +// int numElemToDelete = mapSize - 10000; +// AnalysisLogger.getLogger().debug( +// "DatabaseManagement-> The result size is greater than 10000 rows. Rows number to delete: " +// + numElemToDelete); +// AnalysisLogger.getLogger().debug( +// "DatabaseManagement-> Starting to remove elements"); +// int i=10000; +// if (numElemToDelete != 0){ +// while ((i < mapSize)) { +// mapResult.remove(String.valueOf(i)); +// i++; +// +// } +// +// AnalysisLogger.getLogger().debug( +// "DatabaseManagement-> size of the truncated map without header: " +// + (mapResult.size()-1)); +// } +// +// } +// } + + }catch (Throwable e) { + throw e; + }finally{ + //close the file + if (out!=null){ + out.close(); + out = null; + AnalysisLogger.getLogger().debug( + "In DatabaseManagement-> File closed"); + } + } + // build the Map of Result - buildMapResult(results, DataTypeColumns); + // buildMapResult(results, DataTypeColumns); return results; - } /** @@ -902,6 +947,7 @@ public class DatabaseManagement { // "DatabaseManagement->columns names: " + listKeys); if (result != null && result.size() != 0) { + // // write operation in the file for (int i = 0; i < result.size(); i++) { String RowString = ""; @@ -985,6 +1031,8 @@ public class DatabaseManagement { // mapResult.put(Integer.valueOf(i), RowString); mapResult.put(String.valueOf(i), RowString); + + } } } @@ -1028,190 +1076,222 @@ public class DatabaseManagement { // } } } + + //print check +// AnalysisLogger.getLogger().debug( +// "mapResult size with header: " + mapResult.size()); } - // // write the table result in the file and build the map of results - // private void writeTableIntoFile(List result, - // List DataTypeColumns) throws Exception { - // - // // // file that will contain result - // // BufferedWriter out; - // // // String fileName; - // // // fileName = "./cfg/" + "table.txt"; - // // // fileName = "./files/" + "table.txt"; - // // // fileName = - // // // - // // "/home/loredana/workspace/DatabasesResourcesManagerAlgorithms/cfg/" - // // // + "SampleOnTable.txt"; - // // // fileName = "./files/" + "SmartSampleOnTable.txt"; - // // // File file = new File(fileName); - // // file = new File(FileName); - // // out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream( - // // file), "UTF-8")); - // - // // to get columns names and result - // // write headers in the file - // // to recover columns names - // if (header.equals("")) { - // ArrayList listKeys = new ArrayList( - // ((LinkedHashMap) (result.get(0))).keySet()); - // - // for (int i = 0; i < listKeys.size(); i++) { - // if (i != listKeys.size() - 1) { - // header = header + listKeys.get(i) + ", "; - // } else { - // header = header + listKeys.get(i); - // } - // } - // } - // - // // // print check - // // AnalysisLogger.getLogger().debug( - // // "DatabaseManagement->HEADERS: " + header); - // - // out.write(header); - // out.newLine(); - // mapResult.put("HEADERS", header); - // - // // //print check values - // // AnalysisLogger.getLogger().debug( - // // "DatabaseManagement->columns names: " + listKeys); - // - // if (result != null && result.size() != 0) { - // // // write operation in the file - // for (int i = 0; i < result.size(); i++) { - // String RowString = ""; - // Object element = result.get(i); - // - // // arraylist in which each element is a row result - // ArrayList listvalues = new ArrayList( - // ((LinkedHashMap) element).values()); - // - // // // print check - // // AnalysisLogger.getLogger().debug( - // // "DatabaseManagement->values: " + listvalues); - // - // // each row could have several column values - // Object[] row = listvalues.toArray(); - // if (row.length >= 1) { - // for (int j = 0; j < row.length; j++) { - // if (row[j] == null) { - // row[j] = ""; - // } - // // to parse the obtained results in order to align - // // number - // // values with those of postgres - // String original = row[j].toString(); - // - // // // check value - // // AnalysisLogger.getLogger().debug( - // // "In DatabaseManagement->original value: " - // // + original); - // - // String parsed = "" + row[j]; - // if (original != "") { - // // convert database datatypes to Java datatypes - // if (DataTypeColumns == null - // || DataTypeColumns.size() == 0) - // parsed = convertToJavaType(row[j].getClass() - // .getName(), parsed); - // else - // parsed = convertToJavaType( - // DataTypeColumns.get(j), parsed); - // } - // - // // // check value - // // AnalysisLogger.getLogger().debug( - // // "In DatabaseManagement->row: " + (i + 1) - // // + " column: " + (j + 1) + " value= " - // // + parsed); - // - // // write in a file - // if (j != row.length - 1) { - // // out.write("\"" + parsed + "\""); - // // out.write(","); - // out.write(parsed); - // out.write(","); - // - // // System.out.println("write column : " + j); - // // RowString = RowString + parsed + " "; - // if (j == 0) { - // RowString = parsed; - // } else { - // RowString = RowString + "," + parsed; - // } - // } - // if (j == row.length - 1) { - // // out.write("\"" + parsed + "\""); - // // out.newLine(); - // out.write(parsed); - // out.newLine(); - // - // // to add a row to the map - // if (row.length == 1) { - // RowString = parsed; - // } else { - // RowString = RowString + "," + parsed; - // } - // - // // to add a row to the map - // // RowString = RowString + "," + parsed; - // // mapSampleTableResult.put(String.valueOf(i), - // // RowString); - // - // // // check value row - // // AnalysisLogger.getLogger().debug( - // // "writing the value: " + RowString + " key: " - // // + String.valueOf(i)); - // - // // mapResult.put(Integer.valueOf(i), RowString); - // mapResult.put(String.valueOf(i), RowString); - // } - // } - // } - // // else if (result.size() == 1) { - // // - // // // Object RowElement = (Object) result.get(0); - // // - // // if (row[0] == null) { - // // row[0] = ""; - // // } - // // - // // // to parse the obtained results in order to align - // // // number - // // // values with those of postgres - // // String original = row[0].toString(); - // // - // // // // check value - // // // AnalysisLogger.getLogger().debug( - // // // "In DatabaseManagement->original value: " - // // // + original); - // // - // // String parsed = "" + row[0]; - // // - // // if (original != "") { - // // // convert database datatypes to Java datatypes - // // if (DataTypeColumns == null - // // || DataTypeColumns.size() == 0) - // // parsed = convertToJavaType(row[0].getClass() - // // .getName(), parsed); - // // else - // // parsed = convertToJavaType(DataTypeColumns.get(0), - // // parsed); - // // } - // // - // // out.write(row[0].toString()); - // // out.newLine(); - // // - // // // to add a row to the map - // // mapResult.put(String.valueOf(i), row[0].toString()); - // // - // // } - // } - // } - // // close the file - // out.close(); - // } + // write the table result in the file and build the map of results + private void writeTableIntoFile(List result, + List DataTypeColumns) throws Exception { + + // // file that will contain result + // BufferedWriter out; + // // String fileName; + // // fileName = "./cfg/" + "table.txt"; + // // fileName = "./files/" + "table.txt"; + // // fileName = + // // + // "/home/loredana/workspace/DatabasesResourcesManagerAlgorithms/cfg/" + // // + "SampleOnTable.txt"; + // // fileName = "./files/" + "SmartSampleOnTable.txt"; + // // File file = new File(fileName); + // file = new File(FileName); + // out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + // file), "UTF-8")); + + + try{ + // to get columns names and result + // write headers in the file + // to recover columns names + if (header.equals("")) { + ArrayList listKeys = new ArrayList( + ((LinkedHashMap) (result.get(0))).keySet()); + + for (int i = 0; i < listKeys.size(); i++) { + if (i != listKeys.size() - 1) { + header = header + listKeys.get(i) + ", "; + } else { + header = header + listKeys.get(i); + } + } + } + + // // print check + // AnalysisLogger.getLogger().debug( + // "DatabaseManagement->HEADERS: " + header); + + out.write(header); + out.newLine(); + mapResult.put("HEADERS", header); + + // //print check values + // AnalysisLogger.getLogger().debug( + // "DatabaseManagement->columns names: " + listKeys); + + if (result != null && result.size() != 0) { + // // write operation in the file + for (int i = 0; i < result.size(); i++) { + String RowString = ""; + Object element = result.get(i); + + // arraylist in which each element is a row result + ArrayList listvalues = new ArrayList( + ((LinkedHashMap) element).values()); + + // // print check + // AnalysisLogger.getLogger().debug( + // "DatabaseManagement->values: " + listvalues); + + // each row could have several column values + Object[] row = listvalues.toArray(); + if (row.length >= 1) { + for (int j = 0; j < row.length; j++) { + if (row[j] == null) { + row[j] = ""; + } + // to parse the obtained results in order to align + // number + // values with those of postgres + String original = row[j].toString(); + + // // check value + // AnalysisLogger.getLogger().debug( + // "In DatabaseManagement->original value: " + // + original); + + String parsed = "" + row[j]; + if (original != "") { + // convert database datatypes to Java datatypes + if (DataTypeColumns == null + || DataTypeColumns.size() == 0) + parsed = convertToJavaType(row[j].getClass() + .getName(), parsed); + else + parsed = convertToJavaType( + DataTypeColumns.get(j), parsed); + } + + // // check value + // AnalysisLogger.getLogger().debug( + // "In DatabaseManagement->row: " + (i + 1) + // + " column: " + (j + 1) + " value= " + // + parsed); + + // write in a file + if (j != row.length - 1) { + // out.write("\"" + parsed + "\""); + // out.write(","); + out.write(parsed); + out.write(","); + + // System.out.println("write column : " + j); + // RowString = RowString + parsed + " "; + if (j == 0) { + RowString = parsed; + } else { + RowString = RowString + "," + parsed; + } + } + if (j == row.length - 1) { + // out.write("\"" + parsed + "\""); + // out.newLine(); + out.write(parsed); + out.newLine(); + + // to add a row to the map + if (row.length == 1) { + RowString = parsed; + } else { + RowString = RowString + "," + parsed; + } + + // to add a row to the map + // RowString = RowString + "," + parsed; + // mapSampleTableResult.put(String.valueOf(i), + // RowString); + + // // check value row + // AnalysisLogger.getLogger().debug( + // "writing the value: " + RowString + " key: " + // + String.valueOf(i)); + + // mapResult.put(Integer.valueOf(i), RowString); + + //add in the map only the first 1000 rows if the result list size is greater than 1000 + if (result.size()>1000){ + if(i<1000){ + mapResult.put(String.valueOf(i), RowString); + } + + }else{ + mapResult.put(String.valueOf(i), RowString); + } + } + } + } + // else if (result.size() == 1) { + // + // // Object RowElement = (Object) result.get(0); + // + // if (row[0] == null) { + // row[0] = ""; + // } + // + // // to parse the obtained results in order to align + // // number + // // values with those of postgres + // String original = row[0].toString(); + // + // // // check value + // // AnalysisLogger.getLogger().debug( + // // "In DatabaseManagement->original value: " + // // + original); + // + // String parsed = "" + row[0]; + // + // if (original != "") { + // // convert database datatypes to Java datatypes + // if (DataTypeColumns == null + // || DataTypeColumns.size() == 0) + // parsed = convertToJavaType(row[0].getClass() + // .getName(), parsed); + // else + // parsed = convertToJavaType(DataTypeColumns.get(0), + // parsed); + // } + // + // out.write(row[0].toString()); + // out.newLine(); + // + // // to add a row to the map + // mapResult.put(String.valueOf(i), row[0].toString()); + // + // } + } + } + + AnalysisLogger.getLogger().debug( + "In DatabaseManagement-> map size without header: " + (mapResult.size()-1)); + + AnalysisLogger.getLogger().debug( + "In DatabaseManagement-> Writing File and Result Map creation operations terminated"); + + }catch (Throwable e) { + throw e; + }finally{ + // close the file + if (out!=null){ + out.close(); + out=null; + AnalysisLogger.getLogger().debug( + "In DatabaseManagement-> File closed"); + } + + } + } // to retrieve datatype columns of a table private List getDataTypeColumns(String tableName, String schemaName) @@ -1395,14 +1475,14 @@ public class DatabaseManagement { } - // // to return the file in which the query result (originated from a submit - // // query) is stored - // public File getFileQueryResult() { - // - // // return fileQueryResult; - // return file; - // - // } + // to return the file in which the query result (originated from a submit + // query) is stored + public File getFileQueryResult() { + + // return fileQueryResult; + return file; + + } // // to return the file in which the table result (originated from a submit // // query and sample operations) is stored