gCubeSystem
/
database-resource-manager
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bugs fixed in showcreatetable and samplings operations. In these operation it is useful to insert the table name in a query as "schema.table" for postgres and "db.table" for mysql. 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-access/DatabasesResourcesManager@98907 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Loredana Liccardo 2014-07-23 15:11:58 +00:00
parent aa3fbea54b
commit 229f351f7d
4 changed files with 63 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/main/java/org/gcube/dataaccess/databases/sampler/Sampler.java
View File

@ -18,11 +18,13 @@ import org.hibernate.SessionFactory;
public class Sampler { public class Sampler {
// query to perform sample operation on the table // query to perform sample operation on the table
private static final String queryForSampleOnTablePostgres = "select %1$s from \"%2$s\" limit 100"; // private static final String queryForSampleOnTablePostgres = "select %1$s from \"%2$s\" limit 100";
private static final String queryForSampleOnTablePostgres = "select %1$s from %2$s limit 100";
private static final String queryForSampleOnTableMysql = "select %1$s from %2$s limit 100"; private static final String queryForSampleOnTableMysql = "select %1$s from %2$s limit 100";
// query to perform a smart sample operation randomly on the table // query to perform a smart sample operation randomly on the table
private static final String queryForSmartSampleOnTablePostgres = "select %1$s from \"%2$s\" order by random() limit 200"; // private static final String queryForSmartSampleOnTablePostgres = "select %1$s from \"%2$s\" order by random() limit 200";
private static final String queryForSmartSampleOnTablePostgres = "select %1$s from %2$s order by random() limit 200";
private static final String queryForSmartSampleOnTableMysql = "select %1$s from %2$s order by rand() limit 200"; private static final String queryForSmartSampleOnTableMysql = "select %1$s from %2$s order by rand() limit 200";
// private static final String queryForSmartSampleOnTablePostgres = // private static final String queryForSmartSampleOnTablePostgres =
// "select * from \"%1$s\" order by random() limit 200"; // "select * from \"%1$s\" order by random() limit 200";
@ -31,7 +33,8 @@ public class Sampler {
// query to perform a smart sample operation on the table considering the // query to perform a smart sample operation on the table considering the
// threshold // threshold
private static final String queryForSmartSampleWithThresholdOnTablePostgres = "select %1$s from \"%2$s\" limit 200 offset %3$s"; // private static final String queryForSmartSampleWithThresholdOnTablePostgres = "select %1$s from \"%2$s\" limit 200 offset %3$s";
private static final String queryForSmartSampleWithThresholdOnTablePostgres = "select %1$s from %2$s limit 200 offset %3$s";
// private static final String queryForSmartSampleWithThresholdOnTableMysql // private static final String queryForSmartSampleWithThresholdOnTableMysql
// = "select %1$s from %2$s limit 200 offset %3$s"; // = "select %1$s from %2$s limit 200 offset %3$s";
@ -40,7 +43,8 @@ public class Sampler {
// "select %1$s from \"%2$s\" order by random() limit 100"; // "select %1$s from \"%2$s\" order by random() limit 100";
// query to perform a smart sample operation on the table considering the // query to perform a smart sample operation on the table considering the
// threshold // threshold
private static final String queryForRandomSampleWithThresholdOnTablePostgres = "select %1$s from \"%2$s\" limit 100 offset %3$s"; // private static final String queryForRandomSampleWithThresholdOnTablePostgres = "select %1$s from \"%2$s\" limit 100 offset %3$s";
private static final String queryForRandomSampleWithThresholdOnTablePostgres = "select %1$s from %2$s limit 100 offset %3$s";
private static final String queryForRandomSampleOnTableMysql = "select %1$s from %2$s order by rand() limit 100"; private static final String queryForRandomSampleOnTableMysql = "select %1$s from %2$s order by rand() limit 100";
private static final String queryForRandomSampleOnTablePostgres = "select %1$s from %2$s order by random() limit 100"; private static final String queryForRandomSampleOnTablePostgres = "select %1$s from %2$s order by random() limit 100";
@ -91,7 +95,10 @@ public class Sampler {
// preparing the query // preparing the query
if (DBType.equals(POSTGRES)) { if (DBType.equals(POSTGRES)) {
//the full name equal to "schemaname.tablename"
tableName=schemaName+"."+tableName;
querySampleOnTable = String.format(queryForSampleOnTablePostgres, querySampleOnTable = String.format(queryForSampleOnTablePostgres,
listAttributes, tableName); listAttributes, tableName);
@ -99,6 +106,9 @@ public class Sampler {
if (DBType.equals(MYSQL)) { if (DBType.equals(MYSQL)) {
//the full name equal to "dbname.tablename"
tableName=schemaName+"."+tableName;
querySampleOnTable = String.format(queryForSampleOnTableMysql, querySampleOnTable = String.format(queryForSampleOnTableMysql,
listAttributes, tableName); listAttributes, tableName);
@ -498,13 +508,19 @@ public class Sampler {
// build the query for database postgres // build the query for database postgres
if (DBType.equals(POSTGRES)) { if (DBType.equals(POSTGRES)) {
//the full name equal to "schemaname.tablename"
tablename=schemaName+"."+tablename;
query = String.format(queryForSmartSampleOnTablePostgres, query = String.format(queryForSmartSampleOnTablePostgres,
listAttributes, tablename); listAttributes, tablename);
} }
// build the query for database mysql // build the query for database mysql
if (DBType.equals(MYSQL)) { if (DBType.equals(MYSQL)) {
//the full name equal to "dbname.tablename"
tablename=schemaName+"."+tablename;
query = String.format(queryForSmartSampleOnTableMysql, query = String.format(queryForSmartSampleOnTableMysql,
listAttributes, tablename); listAttributes, tablename);
@ -886,6 +902,9 @@ public class Sampler {
// build the query for database postgres // build the query for database postgres
if (DBType.equals(POSTGRES)) { if (DBType.equals(POSTGRES)) {
//the full name equal to "schemaname.tablename"
tablename=schemaName+"."+tablename;
query = String.format( query = String.format(
queryForSmartSampleWithThresholdOnTablePostgres, queryForSmartSampleWithThresholdOnTablePostgres,
listAttributes, tablename, indexes[i]); listAttributes, tablename, indexes[i]);
@ -1203,6 +1222,9 @@ public class Sampler {
// order to solve a bug with the random function in postgres // order to solve a bug with the random function in postgres
if ((NumRows <= 700000) && (DBType.equals(POSTGRES))) { // Postgres if ((NumRows <= 700000) && (DBType.equals(POSTGRES))) { // Postgres
//the full name equal to "schemaname.tablename"
tableName=schemaName+"."+tableName;
querySampleOnTable = String.format( querySampleOnTable = String.format(
queryForRandomSampleOnTablePostgres, listAttributes, queryForRandomSampleOnTablePostgres, listAttributes,
@ -1211,6 +1233,9 @@ public class Sampler {
} }
if ((NumRows > 700000) && (DBType.equals(POSTGRES))) { // Postgres if ((NumRows > 700000) && (DBType.equals(POSTGRES))) { // Postgres
//the full name equal to "schemaname.tablename"
tableName=schemaName+"."+tableName;
// generate an index randomly to execute the query // generate an index randomly to execute the query
@ -1254,6 +1279,9 @@ public class Sampler {
} }
if (DBType.equals(MYSQL)) { // MySQL if (DBType.equals(MYSQL)) { // MySQL
//the full name equal to "dbname.tablename"
tableName=schemaName+"."+tableName;
querySampleOnTable = String querySampleOnTable = String
.format(queryForRandomSampleOnTableMysql, listAttributes, .format(queryForRandomSampleOnTableMysql, listAttributes,

8
src/main/java/org/gcube/dataaccess/databases/structure/MySQLTableStructure.java
View File

@ -23,6 +23,7 @@ public class MySQLTableStructure extends AbstractTableStructure {
// Retrieve the query // Retrieve the query
String queryForIndexes = getQueryForIndexes(dbSession); String queryForIndexes = getQueryForIndexes(dbSession);
try { try {
@ -35,6 +36,9 @@ public class MySQLTableStructure extends AbstractTableStructure {
// String createTableStatement = (String) (((Object[]) // String createTableStatement = (String) (((Object[])
// indexSet.get(0))[1]); // indexSet.get(0))[1]);
//the full name equal to "dbname.tablename"
tableName = this.databaseName+"."+tableName;
List<Object> result = connection.executeQuery( List<Object> result = connection.executeQuery(
String.format(queryForIndexes, tableName), dbSession); String.format(queryForIndexes, tableName), dbSession);
@ -79,7 +83,9 @@ public class MySQLTableStructure extends AbstractTableStructure {
throws Exception { throws Exception {
// TODO Auto-generated method stub // TODO Auto-generated method stub
String queryForIndexes = "SHOW CREATE TABLE `%1$s`;"; // String queryForIndexes = "SHOW CREATE TABLE `%1$s`;";
String queryForIndexes = "SHOW CREATE TABLE %1$s;";
return queryForIndexes; return queryForIndexes;

9
src/main/java/org/gcube/dataaccess/databases/utils/DatabaseManagement.java
View File

@ -610,12 +610,13 @@ public class DatabaseManagement {
} }
// Method that returns the estimated number of rows // Method that returns the estimated number of rows
public long getNumberOfRows(String tablename) throws Exception { public long getNumberOfRows(String tablename, String schemaName) throws Exception {
long rows; long rows;
rows = op.calculateElements(connection, DBType, tablename, rows = op.calculateElements(connection, DBType, tablename, schemaName,
sourceDBSession); sourceDBSession);
AnalysisLogger.getLogger().debug( AnalysisLogger.getLogger().debug(
"DatabaseManagement->rows' number calculated: " + rows); "DatabaseManagement->rows' number calculated: " + rows);
@ -654,7 +655,7 @@ public class DatabaseManagement {
if (estimatedRows == 0) { if (estimatedRows == 0) {
// estimatedRows = Integer.valueOf(getNumberOfRows(tableName)); // estimatedRows = Integer.valueOf(getNumberOfRows(tableName));
estimatedRows = getNumberOfRows(tableName); estimatedRows = getNumberOfRows(tableName, schemaName);
} }
// to retrieve datatype columns of a table // to retrieve datatype columns of a table
@ -861,7 +862,7 @@ public class DatabaseManagement {
if (estimatedRows == 0) { if (estimatedRows == 0) {
estimatedRows = getNumberOfRows(tableName); estimatedRows = getNumberOfRows(tableName, schemaName);
} }
Sampler sampler = new Sampler(); Sampler sampler = new Sampler();

23
src/main/java/org/gcube/dataaccess/databases/utils/DatabaseOperations.java
View File

@ -21,16 +21,20 @@ public class DatabaseOperations {
// private static final String QueryPostgres = // private static final String QueryPostgres =
// "select count(*) from \"%1$s\" limit 1"; // "select count(*) from \"%1$s\" limit 1";
private static final String QueryPostgres = "select count(*) from (select * from \"%1$s\" limit 1) as a"; // private static final String QueryPostgres = "select count(*) from (select * from \"%1$s\" limit 1) as a";
private static final String QueryMysql = "select count(*) from (select * from `%1$s` limit 1) as a"; private static final String QueryPostgres = "select count(*) from (select * from %1$s limit 1) as a";
// private static final String QueryMysql = "select count(*) from (select * from `%1$s` limit 1) as a";
private static final String QueryMysql = "select count(*) from (select * from %1$s limit 1) as a";
// private static final String QueryMysql = // private static final String QueryMysql =
// "select count(*) from `%1$s` limit 1"; // "select count(*) from `%1$s` limit 1";
// private static final String Query = "select * from %1$s limit 1"; // private static final String Query = "select * from %1$s limit 1";
// private static final String countQuery = "select count(*) from %1$s"; // private static final String countQuery = "select count(*) from %1$s";
// private static final String explainQuery = "explain select * from %1$s"; // private static final String explainQuery = "explain select * from %1$s";
private static final String explainQueryPostgres = "explain select * from \"%1$s\""; // private static final String explainQueryPostgres = "explain select * from \"%1$s\"";
private static final String explainQueryMysql = "explain select * from `%1$s`"; private static final String explainQueryPostgres = "explain select * from %1$s";
// private static final String explainQueryMysql = "explain select * from `%1$s`";
private static final String explainQueryMysql = "explain select * from %1$s";
private static final String MYSQL = "MySQL"; private static final String MYSQL = "MySQL";
private static final String POSTGRES = "Postgres"; private static final String POSTGRES = "Postgres";
@ -104,7 +108,7 @@ public class DatabaseOperations {
// Method that calculate the estimated number of rows // Method that calculate the estimated number of rows
public long calculateElements(ConnectionManager connection, public long calculateElements(ConnectionManager connection,
String dbType, String tablename, SessionFactory session) String dbType, String tablename, String schemaName, SessionFactory session)
throws Exception { throws Exception {
long count = 0; long count = 0;
@ -112,11 +116,16 @@ public class DatabaseOperations {
String countingQuery = null; String countingQuery = null;
if (dbType.equals(POSTGRES)) { if (dbType.equals(POSTGRES)) {
//the full name equal to "schemaname.tablename"
tablename=schemaName+"."+tablename;
countingQuery = String.format(QueryPostgres, tablename); countingQuery = String.format(QueryPostgres, tablename);
} }
if (dbType.equals(MYSQL)) { if (dbType.equals(MYSQL)) {
//the full name equal to "dbname.tablename"
tablename=schemaName+"."+tablename;
countingQuery = String.format(QueryMysql, tablename); countingQuery = String.format(QueryMysql, tablename);
@ -164,12 +173,14 @@ public class DatabaseOperations {
String explain = null; String explain = null;
if (dbType.equals(POSTGRES)) { if (dbType.equals(POSTGRES)) {
explain = String.format(explainQueryPostgres, tablename); explain = String.format(explainQueryPostgres, tablename);
} }
if (dbType.equals(MYSQL)) { if (dbType.equals(MYSQL)) {
explain = String.format(explainQueryMysql, tablename); explain = String.format(explainQueryMysql, tablename);
} }