gCubeSystem
/
document-store-lib-postgresql
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored library

This commit is contained in:
Luca Frosini 2021-03-18 11:24:46 +01:00
parent 7c422ef0eb
commit 4c81b7fdf9
5 changed files with 108 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
README.md
View File

@ -1,71 +0,0 @@
# Document Store Backend Connector Library for PostgreSQL
Document Store Backend Connector Library for PostgreSQL
## Built With
* [OpenJDK](https://openjdk.java.net/) - The JDK used
* [Maven](https://maven.apache.org/) - Dependency Management
## Documentation
[Document Store Backend Connector Library for PostgreSQL](https://wiki.gcube-system.org/gcube/Document_Store_Library)
## Change log
See [Releases](https://code-repo.d4science.org/gCubeSystem/document-store-lib-postgresql/releases).
## Authors
* **Luca Frosini** ([ORCID](https://orcid.org/0000-0003-3183-2291)) - [ISTI-CNR Infrascience Group](http://nemis.isti.cnr.it/groups/infrascience)
## How to Cite this Software
Tell people how to cite this software.
* Cite an associated paper?
* Use a specific BibTeX entry for the software?
@Manual{,
title = {Document Store Backend Connector Library for PostgreSQL},
author = {{Frosini, Luca}},
organization = {ISTI - CNR},
address = {Pisa, Italy},
year = 2021,
url = {http://www.gcube-system.org/}
}
## License
This project is licensed under the EUPL V.1.1 License - see the [LICENSE.md](LICENSE.md) file for details.
## About the gCube Framework
This software is part of the [gCubeFramework](https://www.gcube-system.org/ "gCubeFramework"): an
open-source software toolkit used for building and operating Hybrid Data
Infrastructures enabling the dynamic deployment of Virtual Research Environments
by favouring the realisation of reuse oriented policies.
The projects leading to this software have received funding from a series of European Union programmes including:
- the Sixth Framework Programme for Research and Technological Development
- DILIGENT (grant no. 004260).
- the Seventh Framework Programme for research, technological development and demonstration
- D4Science (grant no. 212488);
- D4Science-II (grant no.239019);
- ENVRI (grant no. 283465);
- iMarine(grant no. 283644);
- EUBrazilOpenBio (grant no. 288754).
- the H2020 research and innovation programme
- SoBigData (grant no. 654024);
- PARTHENOS (grant no. 654119);
- EGIEngage (grant no. 654142);
- ENVRIplus (grant no. 654182);
- BlueBRIDGE (grant no. 675680);
- PerformFish (grant no. 727610);
- AGINFRAplus (grant no. 731001);
- DESIRA (grant no. 818194);
- ARIADNEplus (grant no. 823914);
- RISIS2 (grant no. 824091);

5
pom.xml
View File

@ -46,6 +46,11 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.gcube.accounting</groupId>
<artifactId>accounting-postgresql-utilities</artifactId>
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
<dependency>
<groupId>org.postgresql</groupId>

61
src/main/java/org/gcube/documentstore/persistence/PersistencePostgreSQL.java
View File

@ -4,15 +4,17 @@
package org.gcube.documentstore.persistence;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -21,7 +23,7 @@ import org.slf4j.LoggerFactory;
*/
public class PersistencePostgreSQL extends PersistenceBackend {
private static final Logger logger = LoggerFactory.getLogger(PersistencePostgreSQL.class);
protected static final Logger logger = LoggerFactory.getLogger(PersistencePostgreSQL.class);
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
@ -29,37 +31,29 @@ public class PersistencePostgreSQL extends PersistenceBackend {
public static final String USERNAME_PROPERTY_KEY = "username";
public static final String PASSWORD_PROPERTY_KEY = "password";
private String url;
private String username;
private String password;
private Connection connection;
private Statement statement;
protected StatementMap statementMap;
@Override
protected void prepareConnection(PersistenceBackendConfiguration configuration) throws Exception {
url = configuration.getProperty(URL_PROPERTY_KEY);
username = configuration.getProperty(USERNAME_PROPERTY_KEY);
password = configuration.getProperty(PASSWORD_PROPERTY_KEY);
}
@Override
protected void openConnection() throws Exception {
try {
Class.forName("org.postgresql.Driver");
connection = DriverManager.getConnection(url, username, password);
logger.trace("Database opened successfully");
connection.setAutoCommit(false);
statement = connection.createStatement();
} catch (Exception e) {
throw e;
Map<String, Class<? extends AggregatedRecord<?,?>>> aggregatedRecords = RecordUtility.getAggregatedRecordClassesFound();
for(String typeName : aggregatedRecords.keySet()) {
try {
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
RecordToDBMapping.getRecordToDB(clz);
} catch (Exception e) {
new RuntimeException(e);
}
}
}
public void newConnection() throws Exception {
openConnection();
}
@Override
protected void openConnection() throws Exception {
}
protected void appendString(StringBuffer values, String string) {
values.append("'");
values.append(string);
@ -144,6 +138,7 @@ public class PersistencePostgreSQL extends PersistenceBackend {
@Override
protected void reallyAccount(Record record) throws Exception {
Statement statement = statementMap.getStatement(record);
String sqlCommand = getSQLInsertCommand(record);
statement.executeUpdate(sqlCommand);
}
@ -153,22 +148,22 @@ public class PersistencePostgreSQL extends PersistenceBackend {
}
@Override
protected void clean() throws Exception {}
protected void clean() throws Exception {
statementMap.close();
}
@Override
protected void closeConnection() throws Exception {
statement.close();
connection.commit();
connection.close();
statementMap.close();
}
public void commitAndClose() throws Exception {
closeConnection();
statementMap.close();
}
@Override
public boolean isConnectionActive() throws Exception {
return connection.isValid(300);
return true;
};
}

75
src/main/java/org/gcube/documentstore/persistence/StatementMap.java Normal file
View File

@ -0,0 +1,75 @@
package org.gcube.documentstore.persistence;
import java.sql.Connection;
import java.sql.Statement;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import org.gcube.accounting.utility.postgresql.RecordToDBConnection;
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
import org.gcube.documentstore.records.AggregatedRecord;
import org.gcube.documentstore.records.Record;
import org.gcube.documentstore.records.RecordUtility;
public class StatementMap {
private static final InheritableThreadLocal<Map<String, Statement>> statementsThreadLocal = new InheritableThreadLocal<Map<String, Statement>>() {
@Override
protected Map<String, Statement> initialValue() {
return new HashMap<String, Statement>();
}
};
private PersistenceBackendConfiguration configuration;
public StatementMap(PersistenceBackendConfiguration configuration) {
this.configuration = configuration;
Map<String, Class<? extends AggregatedRecord<?,?>>> aggregatedRecords = RecordUtility.getAggregatedRecordClassesFound();
for(String typeName : aggregatedRecords.keySet()) {
try {
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
RecordToDBMapping.getRecordToDB(clz);
} catch (Exception e) {
new RuntimeException(e);
}
}
}
protected Connection getConnection(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
RecordToDBConnection recordDBInfo = RecordToDBMapping.getRecordDBInfo(clz);
if(recordDBInfo == null) {
RecordToDBMapping.addRecordToDB(clz, configuration);
}
return recordDBInfo.getConnection();
}
public synchronized Statement getStatement(Record record) throws Exception {
Map<String, Statement> map = statementsThreadLocal.get();
String type = record.getRecordType();
Statement statement = map.get(type);
if(statement == null) {
Class<? extends AggregatedRecord<?, ?>> clz = RecordUtility.getAggregatedRecordClass(type);
Connection connection = getConnection(clz);
statement = connection.createStatement();
map.put(type, statement);
}
return statement;
}
public synchronized void close() throws Exception {
Map<String, Statement> map = statementsThreadLocal.get();
Collection<Statement> statements = map.values();
statementsThreadLocal.remove();
for(Statement statement : statements) {
statement.close();
Connection connection = statement.getConnection();
connection.commit();
connection.close();
}
}
}

2
src/test/java/org/gcube/documentstore/persistence/PersistencePostgreSQLTest.java
View File

@ -246,11 +246,9 @@ public class PersistencePostgreSQLTest extends ContextTest {
PersistenceBackend persistenceBackend = PersistenceBackendFactory.rediscoverPersistenceBackend(fallbackPersistenceBackend, context);
Assert.assertTrue(persistenceBackend instanceof PersistencePostgreSQL);
PersistencePostgreSQL persistencePostgreSQL = (PersistencePostgreSQL) persistenceBackend;
persistencePostgreSQL.newConnection();
for(int i=0; i<1000000; i++) {
if(i%50000 == 0) {
persistencePostgreSQL.commitAndClose();
persistencePostgreSQL.newConnection();
}
AggregatedServiceUsageRecord usageRecord = getTestServiceUsageRecord();