gCubeSystem
/
accounting-dashboard-harvester-se-plugin
15
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added auto creation of contexts 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167822 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-29 11:05:55 +00:00
parent 618f18e331
commit 3a69d5e50c
4 changed files with 187 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -64,19 +64,19 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
return properties;
}
public void getConfigParameters() throws IOException {
public Properties getConfigParameters() throws IOException {
Properties properties = new Properties();
try {
Properties properties = new Properties();
InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader()
.getResourceAsStream(PROPERTY_FILENAME);
properties.load(input);
getProperties().set(properties);
return properties;
} catch(Exception e) {
logger.warn(
"Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults",
PROPERTY_FILENAME);
}
return properties;
}
/** {@inheritDoc} */
@ -124,7 +124,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.debug("Harvesting from {} to {}", DateUtils.format(start),
DateUtils.format(end));
getConfigParameters();
Properties properties = getConfigParameters();
getProperties().set(properties);
ContextAuthorization contextAuthorization = new ContextAuthorization();

140
src/main/java/org/gcube/dataharvest/dao/Dao.java
View File

@ -7,8 +7,10 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.DateUtils;
import org.postgresql.util.PSQLException;
@ -32,7 +34,7 @@ public class Dao {
public void init() throws DaoException {
try {
Class.forName("org.postgresql.Driver");
} catch (ClassNotFoundException ex) {
} catch(ClassNotFoundException ex) {
logger.error(ex.getLocalizedMessage());
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
}
@ -50,7 +52,7 @@ public class Dao {
try {
conn = DriverManager.getConnection(url, user, password);
logger.debug("Connected to: " + url);
} catch (SQLException ex) {
} catch(SQLException ex) {
logger.error(ex.getLocalizedMessage());
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
}
@ -63,12 +65,12 @@ public class Dao {
*/
public void disconnect() throws DaoException {
try {
if (conn != null) {
if(conn != null) {
conn.close();
conn = null;
logger.debug("Disconnecting from database");
}
} catch (SQLException ex) {
} catch(SQLException ex) {
logger.error(ex.getLocalizedMessage());
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
}
@ -106,26 +108,26 @@ public class Dao {
try {
s = conn.createStatement();
rs = s.executeQuery(query);
while (rs.next()) {
while(rs.next()) {
list.add(rs.getString("dname"));
}
return list.toArray(new String[list.size()]);
} catch (Exception ex) {
} catch(Exception ex) {
logger.error(ex.getLocalizedMessage());
return null;
} finally {
if (rs != null) {
if(rs != null) {
try {
rs.close();
} catch (SQLException ex) {
} catch(SQLException ex) {
// do nothing
}
}
if (s != null) {
if(s != null) {
try {
s.close();
} catch (SQLException ex) {
} catch(SQLException ex) {
// do nothing
}
}
@ -140,7 +142,7 @@ public class Dao {
* @param to
* @throws DaoException
*/
public void insertMonthlyMeasure(List<HarvestedData> data, java.util.Date from, java.util.Date to, boolean doUpdate)
public void insertMonthlyMeasure(List<HarvestedData> data, Date from, Date to, boolean doUpdate)
throws DaoException {
// first of all: check if data of the same type are already in the
// database.
@ -152,13 +154,13 @@ public class Dao {
int monthTo = cTo.get(Calendar.MONTH);
int yearTo = cTo.get(Calendar.YEAR);
if ((monthFrom != monthTo) || (yearFrom != yearTo)) {
if((monthFrom != monthTo) || (yearFrom != yearTo)) {
String err = "Invalid time period. The time period MUST refer one month.";
logger.error(err);
throw new DaoException(err, null);
}
if (data == null || data.size() <= 0) {
if(data == null || data.size() <= 0) {
String err = "No data passed in input. Aborting operation.";
logger.error(err);
throw new DaoException(err, null);
@ -166,33 +168,34 @@ public class Dao {
monthFrom++; // because january = 0...
try {
for (HarvestedData harvest : data) {
String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType()
+ " and context_id=(select id from context where dname='" + harvest.getContext()
+ "') and month=" + monthFrom + " and year=" + yearFrom;
for(HarvestedData harvestedData : data) {
int contextId = getOrInsertContextId(harvestedData.getContext());
String query = "select id from monthly_measure where measure_type_id=" + harvestedData.getDataType()
+ " and context_id=" + contextId + " and month=" + monthFrom + " and year=" + yearFrom;
logger.debug(query);
Statement sel = conn.createStatement();
ResultSet rs = sel.executeQuery(query);
if (rs.next()) {
if (doUpdate) {
if(rs.next()) {
if(doUpdate) {
// record found: update it
Statement s = conn.createStatement();
int id = rs.getInt("id");
String update = "update monthly_measure set measure=" + harvest.getMeasure() + " where id="
String update = "update monthly_measure set measure=" + harvestedData.getMeasure() + " where id="
+ id;
logger.debug(update);
s.execute(update);
s.close();
} else {
logger.warn("Skipped " + harvest.getContext());
logger.warn("Skipped " + harvestedData.getContext());
}
} else {
// record not found: insert new record
Statement s = conn.createStatement();
String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values (";
insert += yearFrom + "," + monthFrom + "," + harvest.getMeasure() + "," + harvest.getDataType()
insert += yearFrom + "," + monthFrom + "," + harvestedData.getMeasure() + "," + harvestedData.getDataType()
+ ",";
insert += "(select id from context where dname='" + harvest.getContext() + "'),";
insert += "(select id from context where dname='" + harvestedData.getContext() + "'),";
insert += "'" + yearFrom + "-" + monthFrom + "-01'";
insert += ")";
logger.debug(insert);
@ -204,16 +207,73 @@ public class Dao {
sel.close();
}
} catch (PSQLException x) {
} catch(PSQLException x) {
// handled exception: try to iterate...
logger.error(x.getLocalizedMessage());
} catch (Exception x) {
} catch(Exception x) {
// not handled exception: stop
logger.error(x.getLocalizedMessage());
throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x);
}
}
public enum contextType {
INFRASTRUCTURE, VO, VRE
}
public int getOrInsertContextId(String contextFullName) throws SQLException {
String query = "select id from context where dname='" + contextFullName + "'";
logger.debug(query);
Statement sel = conn.createStatement();
ResultSet rs = sel.executeQuery(query);
if(rs.next()) {
// context found
int id = rs.getInt("id");
logger.debug("Context {} has id {}", contextFullName, id);
return id;
} else {
// context not found: insert new record
int parentId = 0; // The id of D4Science infrastructure as aggregator
int lastIndexOfSlash = contextFullName.lastIndexOf("/");
if(lastIndexOfSlash!=0) {
String parentContextFullName = contextFullName.substring(0, contextFullName.lastIndexOf("/"));
parentId = getOrInsertContextId(parentContextFullName);
}
// It is a new context and we don't know which context type is. Using 0 for ROOT, 1 for VO, 2 for VRE
int contextTypeId = contextType.INFRASTRUCTURE.ordinal();
ScopeBean scopeBean = new ScopeBean(contextFullName);
switch(scopeBean.type()) {
case INFRASTRUCTURE:
contextTypeId = contextType.INFRASTRUCTURE.ordinal();
break;
case VO:
contextTypeId = contextType.VO.ordinal();
break;
case VRE:
contextTypeId = contextType.VRE.ordinal();
break;
default:
break;
}
Statement s = conn.createStatement();
String insert = "insert into context (name, context_id, context_type_id, dname) values ('";
insert += contextFullName + "'," + parentId + "," + contextTypeId + ", '" + contextFullName;
insert += "')";
logger.debug(insert);
s.execute(insert);
s.close();
}
rs.close();
sel.close();
return getOrInsertContextId(contextFullName);
}
public ArrayList<Integer> getSubTree(Integer rootId) throws DaoException {
String queryBase = "select id from context where context_id in (%%)";
ArrayList<Integer> subTree = new ArrayList<>();
@ -223,10 +283,10 @@ public class Dao {
temp.add(rootId);
subTree.add(rootId);
boolean again = true;
for (int i = 0; (i < 10) && (again); i++) {
for(int i = 0; (i < 10) && (again); i++) {
try {
String listId = "";
for (Integer id : temp) {
for(Integer id : temp) {
listId += "," + id;
}
listId = listId.substring(1);
@ -234,23 +294,22 @@ public class Dao {
String query = queryBase.replace("%%", listId);
s = conn.createStatement();
rs = s.executeQuery(query);
if (rs.next()) {
if(rs.next()) {
temp = new ArrayList<>();
Integer dbId = rs.getInt("id");
subTree.add(dbId);
temp.add(dbId);
while (rs.next()) {
while(rs.next()) {
dbId = rs.getInt("id");
subTree.add(dbId);
temp.add(dbId);
}
}
else {
} else {
again = false;
}
rs.close();
s.close();
} catch (Exception x) {
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x);
}
@ -271,7 +330,7 @@ public class Dao {
rep.execute(report);
String query = "select id from report where name=\'VRE Social\' order by id desc";
rs = rep.executeQuery(query);
if (rs.next()) {
if(rs.next()) {
int id = rs.getInt("id");
String reportItem = "insert into report_item (report_id, type_id_1, type_id_1_name) ";
reportItem += "values (" + id + ", 8, \'mt\'), (" + id + ", 9, \'mt\'), (" + id + ", 10, \'mt\')";
@ -279,24 +338,23 @@ public class Dao {
} else
throw new DaoException("No report id.", null);
} catch (SQLException ex) {
} catch(SQLException ex) {
logger.error(ex.getLocalizedMessage());
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
} finally {
try {
if (rs != null) {
if(rs != null) {
rs.close();
}
if (rep != null) {
if(rep != null) {
rep.close();
}
} catch (SQLException ex) {
} catch(SQLException ex) {
logger.error(ex.getLocalizedMessage());
}
}
}
/**
* Dummy tester
*
@ -312,14 +370,14 @@ public class Dao {
try {
s = conn.createStatement();
s.execute(insert);
} catch (SQLException ex) {
} catch(SQLException ex) {
logger.error(ex.getLocalizedMessage());
throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
} finally {
if (s != null)
if(s != null)
try {
s.close();
} catch (SQLException ex) {
} catch(SQLException ex) {
logger.error(ex.getLocalizedMessage());
}
}

7
src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java
View File

@ -56,6 +56,7 @@ public class DatabaseParameterRetriever {
String password = "";
if(localDB) {
logger.debug("Using configuration from local config");
uri = properties.getProperty(DB_URI);
username = properties.getProperty(DB_USERNAME);
password = properties.getProperty(DB_PASSWORD);
@ -72,8 +73,12 @@ public class DatabaseParameterRetriever {
DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> serviceEndpoints = client.submit(query);
if(serviceEndpoints.size()==0) {
throw new DaoException("No endpoints found to get database connection.");
}
if(serviceEndpoints.size() > 1) {
throw new DaoException("More than one endpoint found. Not sure which one use.");
throw new DaoException("More than one endpoint found to get database connection. Not sure which one use.");
}
Group<AccessPoint> accessPoints = serviceEndpoints.get(0).profile().accessPoints();

54
src/test/java/org/gcube/dataharvest/dao/DaoTests.java Normal file
View File

@ -0,0 +1,54 @@
package org.gcube.dataharvest.dao;
import java.util.LinkedHashMap;
import java.util.Properties;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.DataHarvestPluginDeclaration;
import org.gcube.dataharvest.utils.ContextTest;
import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DaoTests extends ContextTest {
private static Logger logger = LoggerFactory.getLogger(DaoTests.class);
public static SortedSet<String> getContexts() throws Exception{
SortedSet<String> contexts = new TreeSet<>();
LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
for(String scope : map.keySet()) {
try {
String context = map.get(scope).toString();
contexts.add(context);
}catch (Exception e) {
throw e;
}
}
return contexts;
}
@Test
public void testInsertMissingContext() throws Exception {
AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(new DataHarvestPluginDeclaration());
Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
AccountingDataHarvesterPlugin.getProperties().set(properties);
DatabaseManager dbaseManager = new DatabaseManager();
Dao dao = dbaseManager.dbConnect();
String[] contexts = new String[]{"/d4science.research-infrastructures.eu", "/d4science.research-infrastructures.eu/gCubeApps/rScience", "/d4science.research-infrastructures.eu/gCubeApps"};
for(String contextFullname : contexts) {
int id = dao.getOrInsertContextId(contextFullname);
logger.debug("{} is is {}", contextFullname, id);
}
}
}