Refactored library
This commit is contained in:
parent
7c422ef0eb
commit
4c81b7fdf9
71
README.md
71
README.md
|
@ -1,71 +0,0 @@
|
||||||
# Document Store Backend Connector Library for PostgreSQL
|
|
||||||
|
|
||||||
Document Store Backend Connector Library for PostgreSQL
|
|
||||||
|
|
||||||
## Built With
|
|
||||||
|
|
||||||
* [OpenJDK](https://openjdk.java.net/) - The JDK used
|
|
||||||
* [Maven](https://maven.apache.org/) - Dependency Management
|
|
||||||
|
|
||||||
## Documentation
|
|
||||||
|
|
||||||
[Document Store Backend Connector Library for PostgreSQL](https://wiki.gcube-system.org/gcube/Document_Store_Library)
|
|
||||||
|
|
||||||
## Change log
|
|
||||||
|
|
||||||
See [Releases](https://code-repo.d4science.org/gCubeSystem/document-store-lib-postgresql/releases).
|
|
||||||
|
|
||||||
## Authors
|
|
||||||
|
|
||||||
* **Luca Frosini** ([ORCID](https://orcid.org/0000-0003-3183-2291)) - [ISTI-CNR Infrascience Group](http://nemis.isti.cnr.it/groups/infrascience)
|
|
||||||
|
|
||||||
## How to Cite this Software
|
|
||||||
|
|
||||||
Tell people how to cite this software.
|
|
||||||
* Cite an associated paper?
|
|
||||||
* Use a specific BibTeX entry for the software?
|
|
||||||
|
|
||||||
|
|
||||||
@Manual{,
|
|
||||||
title = {Document Store Backend Connector Library for PostgreSQL},
|
|
||||||
author = {{Frosini, Luca}},
|
|
||||||
organization = {ISTI - CNR},
|
|
||||||
address = {Pisa, Italy},
|
|
||||||
year = 2021,
|
|
||||||
url = {http://www.gcube-system.org/}
|
|
||||||
}
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
This project is licensed under the EUPL V.1.1 License - see the [LICENSE.md](LICENSE.md) file for details.
|
|
||||||
|
|
||||||
|
|
||||||
## About the gCube Framework
|
|
||||||
This software is part of the [gCubeFramework](https://www.gcube-system.org/ "gCubeFramework"): an
|
|
||||||
open-source software toolkit used for building and operating Hybrid Data
|
|
||||||
Infrastructures enabling the dynamic deployment of Virtual Research Environments
|
|
||||||
by favouring the realisation of reuse oriented policies.
|
|
||||||
|
|
||||||
The projects leading to this software have received funding from a series of European Union programmes including:
|
|
||||||
|
|
||||||
- the Sixth Framework Programme for Research and Technological Development
|
|
||||||
- DILIGENT (grant no. 004260).
|
|
||||||
- the Seventh Framework Programme for research, technological development and demonstration
|
|
||||||
- D4Science (grant no. 212488);
|
|
||||||
- D4Science-II (grant no.239019);
|
|
||||||
- ENVRI (grant no. 283465);
|
|
||||||
- iMarine(grant no. 283644);
|
|
||||||
- EUBrazilOpenBio (grant no. 288754).
|
|
||||||
- the H2020 research and innovation programme
|
|
||||||
- SoBigData (grant no. 654024);
|
|
||||||
- PARTHENOS (grant no. 654119);
|
|
||||||
- EGIEngage (grant no. 654142);
|
|
||||||
- ENVRIplus (grant no. 654182);
|
|
||||||
- BlueBRIDGE (grant no. 675680);
|
|
||||||
- PerformFish (grant no. 727610);
|
|
||||||
- AGINFRAplus (grant no. 731001);
|
|
||||||
- DESIRA (grant no. 818194);
|
|
||||||
- ARIADNEplus (grant no. 823914);
|
|
||||||
- RISIS2 (grant no. 824091);
|
|
||||||
|
|
||||||
|
|
5
pom.xml
5
pom.xml
|
@ -46,6 +46,11 @@
|
||||||
<groupId>org.slf4j</groupId>
|
<groupId>org.slf4j</groupId>
|
||||||
<artifactId>slf4j-api</artifactId>
|
<artifactId>slf4j-api</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.accounting</groupId>
|
||||||
|
<artifactId>accounting-postgresql-utilities</artifactId>
|
||||||
|
<version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
|
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.postgresql</groupId>
|
<groupId>org.postgresql</groupId>
|
||||||
|
|
|
@ -4,15 +4,17 @@
|
||||||
package org.gcube.documentstore.persistence;
|
package org.gcube.documentstore.persistence;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.sql.Connection;
|
|
||||||
import java.sql.DriverManager;
|
|
||||||
import java.sql.Statement;
|
import java.sql.Statement;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.SortedSet;
|
import java.util.SortedSet;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
|
|
||||||
|
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
|
||||||
|
import org.gcube.documentstore.records.AggregatedRecord;
|
||||||
import org.gcube.documentstore.records.Record;
|
import org.gcube.documentstore.records.Record;
|
||||||
|
import org.gcube.documentstore.records.RecordUtility;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -21,7 +23,7 @@ import org.slf4j.LoggerFactory;
|
||||||
*/
|
*/
|
||||||
public class PersistencePostgreSQL extends PersistenceBackend {
|
public class PersistencePostgreSQL extends PersistenceBackend {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(PersistencePostgreSQL.class);
|
protected static final Logger logger = LoggerFactory.getLogger(PersistencePostgreSQL.class);
|
||||||
|
|
||||||
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
|
public static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS Z";
|
||||||
|
|
||||||
|
@ -29,35 +31,27 @@ public class PersistencePostgreSQL extends PersistenceBackend {
|
||||||
public static final String USERNAME_PROPERTY_KEY = "username";
|
public static final String USERNAME_PROPERTY_KEY = "username";
|
||||||
public static final String PASSWORD_PROPERTY_KEY = "password";
|
public static final String PASSWORD_PROPERTY_KEY = "password";
|
||||||
|
|
||||||
private String url;
|
protected StatementMap statementMap;
|
||||||
private String username;
|
|
||||||
private String password;
|
|
||||||
|
|
||||||
private Connection connection;
|
|
||||||
private Statement statement;
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void prepareConnection(PersistenceBackendConfiguration configuration) throws Exception {
|
protected void prepareConnection(PersistenceBackendConfiguration configuration) throws Exception {
|
||||||
url = configuration.getProperty(URL_PROPERTY_KEY);
|
|
||||||
username = configuration.getProperty(USERNAME_PROPERTY_KEY);
|
|
||||||
password = configuration.getProperty(PASSWORD_PROPERTY_KEY);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
Map<String, Class<? extends AggregatedRecord<?,?>>> aggregatedRecords = RecordUtility.getAggregatedRecordClassesFound();
|
||||||
protected void openConnection() throws Exception {
|
for(String typeName : aggregatedRecords.keySet()) {
|
||||||
try {
|
try {
|
||||||
Class.forName("org.postgresql.Driver");
|
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
|
||||||
connection = DriverManager.getConnection(url, username, password);
|
RecordToDBMapping.getRecordToDB(clz);
|
||||||
logger.trace("Database opened successfully");
|
} catch (Exception e) {
|
||||||
connection.setAutoCommit(false);
|
new RuntimeException(e);
|
||||||
statement = connection.createStatement();
|
}
|
||||||
} catch (Exception e) {
|
|
||||||
throw e;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void newConnection() throws Exception {
|
|
||||||
openConnection();
|
|
||||||
|
@Override
|
||||||
|
protected void openConnection() throws Exception {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void appendString(StringBuffer values, String string) {
|
protected void appendString(StringBuffer values, String string) {
|
||||||
|
@ -144,6 +138,7 @@ public class PersistencePostgreSQL extends PersistenceBackend {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void reallyAccount(Record record) throws Exception {
|
protected void reallyAccount(Record record) throws Exception {
|
||||||
|
Statement statement = statementMap.getStatement(record);
|
||||||
String sqlCommand = getSQLInsertCommand(record);
|
String sqlCommand = getSQLInsertCommand(record);
|
||||||
statement.executeUpdate(sqlCommand);
|
statement.executeUpdate(sqlCommand);
|
||||||
}
|
}
|
||||||
|
@ -153,22 +148,22 @@ public class PersistencePostgreSQL extends PersistenceBackend {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void clean() throws Exception {}
|
protected void clean() throws Exception {
|
||||||
|
statementMap.close();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void closeConnection() throws Exception {
|
protected void closeConnection() throws Exception {
|
||||||
statement.close();
|
statementMap.close();
|
||||||
connection.commit();
|
|
||||||
connection.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void commitAndClose() throws Exception {
|
public void commitAndClose() throws Exception {
|
||||||
closeConnection();
|
statementMap.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isConnectionActive() throws Exception {
|
public boolean isConnectionActive() throws Exception {
|
||||||
return connection.isValid(300);
|
return true;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
|
@ -0,0 +1,75 @@
|
||||||
|
package org.gcube.documentstore.persistence;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.gcube.accounting.utility.postgresql.RecordToDBConnection;
|
||||||
|
import org.gcube.accounting.utility.postgresql.RecordToDBMapping;
|
||||||
|
import org.gcube.documentstore.records.AggregatedRecord;
|
||||||
|
import org.gcube.documentstore.records.Record;
|
||||||
|
import org.gcube.documentstore.records.RecordUtility;
|
||||||
|
|
||||||
|
public class StatementMap {
|
||||||
|
|
||||||
|
private static final InheritableThreadLocal<Map<String, Statement>> statementsThreadLocal = new InheritableThreadLocal<Map<String, Statement>>() {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Map<String, Statement> initialValue() {
|
||||||
|
return new HashMap<String, Statement>();
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
private PersistenceBackendConfiguration configuration;
|
||||||
|
|
||||||
|
public StatementMap(PersistenceBackendConfiguration configuration) {
|
||||||
|
this.configuration = configuration;
|
||||||
|
Map<String, Class<? extends AggregatedRecord<?,?>>> aggregatedRecords = RecordUtility.getAggregatedRecordClassesFound();
|
||||||
|
for(String typeName : aggregatedRecords.keySet()) {
|
||||||
|
try {
|
||||||
|
Class<? extends AggregatedRecord<?,?>> clz = aggregatedRecords.get(typeName);
|
||||||
|
RecordToDBMapping.getRecordToDB(clz);
|
||||||
|
} catch (Exception e) {
|
||||||
|
new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Connection getConnection(Class<? extends AggregatedRecord<?, ?>> clz) throws Exception {
|
||||||
|
RecordToDBConnection recordDBInfo = RecordToDBMapping.getRecordDBInfo(clz);
|
||||||
|
if(recordDBInfo == null) {
|
||||||
|
RecordToDBMapping.addRecordToDB(clz, configuration);
|
||||||
|
}
|
||||||
|
return recordDBInfo.getConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized Statement getStatement(Record record) throws Exception {
|
||||||
|
Map<String, Statement> map = statementsThreadLocal.get();
|
||||||
|
String type = record.getRecordType();
|
||||||
|
Statement statement = map.get(type);
|
||||||
|
if(statement == null) {
|
||||||
|
Class<? extends AggregatedRecord<?, ?>> clz = RecordUtility.getAggregatedRecordClass(type);
|
||||||
|
Connection connection = getConnection(clz);
|
||||||
|
statement = connection.createStatement();
|
||||||
|
map.put(type, statement);
|
||||||
|
}
|
||||||
|
return statement;
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized void close() throws Exception {
|
||||||
|
Map<String, Statement> map = statementsThreadLocal.get();
|
||||||
|
Collection<Statement> statements = map.values();
|
||||||
|
statementsThreadLocal.remove();
|
||||||
|
for(Statement statement : statements) {
|
||||||
|
statement.close();
|
||||||
|
Connection connection = statement.getConnection();
|
||||||
|
connection.commit();
|
||||||
|
connection.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -246,11 +246,9 @@ public class PersistencePostgreSQLTest extends ContextTest {
|
||||||
PersistenceBackend persistenceBackend = PersistenceBackendFactory.rediscoverPersistenceBackend(fallbackPersistenceBackend, context);
|
PersistenceBackend persistenceBackend = PersistenceBackendFactory.rediscoverPersistenceBackend(fallbackPersistenceBackend, context);
|
||||||
Assert.assertTrue(persistenceBackend instanceof PersistencePostgreSQL);
|
Assert.assertTrue(persistenceBackend instanceof PersistencePostgreSQL);
|
||||||
PersistencePostgreSQL persistencePostgreSQL = (PersistencePostgreSQL) persistenceBackend;
|
PersistencePostgreSQL persistencePostgreSQL = (PersistencePostgreSQL) persistenceBackend;
|
||||||
persistencePostgreSQL.newConnection();
|
|
||||||
for(int i=0; i<1000000; i++) {
|
for(int i=0; i<1000000; i++) {
|
||||||
if(i%50000 == 0) {
|
if(i%50000 == 0) {
|
||||||
persistencePostgreSQL.commitAndClose();
|
persistencePostgreSQL.commitAndClose();
|
||||||
persistencePostgreSQL.newConnection();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
AggregatedServiceUsageRecord usageRecord = getTestServiceUsageRecord();
|
AggregatedServiceUsageRecord usageRecord = getTestServiceUsageRecord();
|
||||||
|
|
Loading…
Reference in New Issue