diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index 68f0f37..a24c11f 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -64,19 +64,19 @@ public class AccountingDataHarvesterPlugin extends Plugin data, java.util.Date from, java.util.Date to, boolean doUpdate) + public void insertMonthlyMeasure(List data, Date from, Date to, boolean doUpdate) throws DaoException { // first of all: check if data of the same type are already in the // database. @@ -151,69 +153,127 @@ public class Dao { int yearFrom = cFrom.get(Calendar.YEAR); int monthTo = cTo.get(Calendar.MONTH); int yearTo = cTo.get(Calendar.YEAR); - - if ((monthFrom != monthTo) || (yearFrom != yearTo)) { + + if((monthFrom != monthTo) || (yearFrom != yearTo)) { String err = "Invalid time period. The time period MUST refer one month."; logger.error(err); throw new DaoException(err, null); } - - if (data == null || data.size() <= 0) { + + if(data == null || data.size() <= 0) { String err = "No data passed in input. Aborting operation."; logger.error(err); throw new DaoException(err, null); } monthFrom++; // because january = 0... - + try { - for (HarvestedData harvest : data) { - String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType() - + " and context_id=(select id from context where dname='" + harvest.getContext() - + "') and month=" + monthFrom + " and year=" + yearFrom; + for(HarvestedData harvestedData : data) { + int contextId = getOrInsertContextId(harvestedData.getContext()); + + String query = "select id from monthly_measure where measure_type_id=" + harvestedData.getDataType() + + " and context_id=" + contextId + " and month=" + monthFrom + " and year=" + yearFrom; logger.debug(query); Statement sel = conn.createStatement(); ResultSet rs = sel.executeQuery(query); - if (rs.next()) { - if (doUpdate) { + if(rs.next()) { + if(doUpdate) { // record found: update it Statement s = conn.createStatement(); int id = rs.getInt("id"); - String update = "update monthly_measure set measure=" + harvest.getMeasure() + " where id=" + String update = "update monthly_measure set measure=" + harvestedData.getMeasure() + " where id=" + id; logger.debug(update); s.execute(update); s.close(); } else { - logger.warn("Skipped " + harvest.getContext()); + logger.warn("Skipped " + harvestedData.getContext()); } } else { // record not found: insert new record Statement s = conn.createStatement(); String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values ("; - insert += yearFrom + "," + monthFrom + "," + harvest.getMeasure() + "," + harvest.getDataType() + insert += yearFrom + "," + monthFrom + "," + harvestedData.getMeasure() + "," + harvestedData.getDataType() + ","; - insert += "(select id from context where dname='" + harvest.getContext() + "'),"; + insert += "(select id from context where dname='" + harvestedData.getContext() + "'),"; insert += "'" + yearFrom + "-" + monthFrom + "-01'"; insert += ")"; logger.debug(insert); s.execute(insert); s.close(); } - + rs.close(); sel.close(); - + } - } catch (PSQLException x) { + } catch(PSQLException x) { // handled exception: try to iterate... logger.error(x.getLocalizedMessage()); - } catch (Exception x) { + } catch(Exception x) { // not handled exception: stop logger.error(x.getLocalizedMessage()); throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x); } } - + + public enum contextType { + INFRASTRUCTURE, VO, VRE + } + + public int getOrInsertContextId(String contextFullName) throws SQLException { + String query = "select id from context where dname='" + contextFullName + "'"; + logger.debug(query); + Statement sel = conn.createStatement(); + ResultSet rs = sel.executeQuery(query); + if(rs.next()) { + // context found + int id = rs.getInt("id"); + logger.debug("Context {} has id {}", contextFullName, id); + return id; + } else { + // context not found: insert new record + int parentId = 0; // The id of D4Science infrastructure as aggregator + int lastIndexOfSlash = contextFullName.lastIndexOf("/"); + if(lastIndexOfSlash!=0) { + String parentContextFullName = contextFullName.substring(0, contextFullName.lastIndexOf("/")); + parentId = getOrInsertContextId(parentContextFullName); + } + + // It is a new context and we don't know which context type is. Using 0 for ROOT, 1 for VO, 2 for VRE + int contextTypeId = contextType.INFRASTRUCTURE.ordinal(); + ScopeBean scopeBean = new ScopeBean(contextFullName); + switch(scopeBean.type()) { + case INFRASTRUCTURE: + contextTypeId = contextType.INFRASTRUCTURE.ordinal(); + break; + case VO: + contextTypeId = contextType.VO.ordinal(); + break; + case VRE: + contextTypeId = contextType.VRE.ordinal(); + break; + default: + break; + } + + Statement s = conn.createStatement(); + String insert = "insert into context (name, context_id, context_type_id, dname) values ('"; + insert += contextFullName + "'," + parentId + "," + contextTypeId + ", '" + contextFullName; + insert += "')"; + logger.debug(insert); + + s.execute(insert); + s.close(); + + } + + rs.close(); + sel.close(); + + return getOrInsertContextId(contextFullName); + } + public ArrayList getSubTree(Integer rootId) throws DaoException { String queryBase = "select id from context where context_id in (%%)"; ArrayList subTree = new ArrayList<>(); @@ -223,34 +283,33 @@ public class Dao { temp.add(rootId); subTree.add(rootId); boolean again = true; - for (int i = 0; (i < 10) && (again); i++) { + for(int i = 0; (i < 10) && (again); i++) { try { String listId = ""; - for (Integer id : temp) { + for(Integer id : temp) { listId += "," + id; } listId = listId.substring(1); - + String query = queryBase.replace("%%", listId); s = conn.createStatement(); rs = s.executeQuery(query); - if (rs.next()) { + if(rs.next()) { temp = new ArrayList<>(); - Integer dbId = rs.getInt("id"); + Integer dbId = rs.getInt("id"); subTree.add(dbId); temp.add(dbId); - while (rs.next()) { - dbId = rs.getInt("id"); + while(rs.next()) { + dbId = rs.getInt("id"); subTree.add(dbId); temp.add(dbId); } - } - else { + } else { again = false; - } + } rs.close(); s.close(); - } catch (Exception x) { + } catch(Exception x) { logger.error(x.getLocalizedMessage()); throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x); } @@ -258,7 +317,7 @@ public class Dao { return subTree; } - + public void createSocialReport(int contextId, int orderBy) throws DaoException { Statement rep = null; ResultSet rs = null; @@ -271,32 +330,31 @@ public class Dao { rep.execute(report); String query = "select id from report where name=\'VRE Social\' order by id desc"; rs = rep.executeQuery(query); - if (rs.next()) { + if(rs.next()) { int id = rs.getInt("id"); String reportItem = "insert into report_item (report_id, type_id_1, type_id_1_name) "; reportItem += "values (" + id + ", 8, \'mt\'), (" + id + ", 9, \'mt\'), (" + id + ", 10, \'mt\')"; rep.execute(reportItem); } else throw new DaoException("No report id.", null); - - } catch (SQLException ex) { + + } catch(SQLException ex) { logger.error(ex.getLocalizedMessage()); throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); } finally { try { - if (rs != null) { + if(rs != null) { rs.close(); } - if (rep != null) { + if(rep != null) { rep.close(); } - } catch (SQLException ex) { + } catch(SQLException ex) { logger.error(ex.getLocalizedMessage()); } } } - /** * Dummy tester * @@ -312,17 +370,17 @@ public class Dao { try { s = conn.createStatement(); s.execute(insert); - } catch (SQLException ex) { + } catch(SQLException ex) { logger.error(ex.getLocalizedMessage()); throw new DaoException(ex.getLocalizedMessage(), ex.getCause()); } finally { - if (s != null) + if(s != null) try { s.close(); - } catch (SQLException ex) { + } catch(SQLException ex) { logger.error(ex.getLocalizedMessage()); } } } - + } diff --git a/src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java b/src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java index f56df36..95499ab 100644 --- a/src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java +++ b/src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java @@ -56,6 +56,7 @@ public class DatabaseParameterRetriever { String password = ""; if(localDB) { + logger.debug("Using configuration from local config"); uri = properties.getProperty(DB_URI); username = properties.getProperty(DB_USERNAME); password = properties.getProperty(DB_PASSWORD); @@ -72,8 +73,12 @@ public class DatabaseParameterRetriever { DiscoveryClient client = ICFactory.clientFor(ServiceEndpoint.class); List serviceEndpoints = client.submit(query); + if(serviceEndpoints.size()==0) { + throw new DaoException("No endpoints found to get database connection."); + } + if(serviceEndpoints.size() > 1) { - throw new DaoException("More than one endpoint found. Not sure which one use."); + throw new DaoException("More than one endpoint found to get database connection. Not sure which one use."); } Group accessPoints = serviceEndpoints.get(0).profile().accessPoints(); diff --git a/src/test/java/org/gcube/dataharvest/dao/DaoTests.java b/src/test/java/org/gcube/dataharvest/dao/DaoTests.java new file mode 100644 index 0000000..9fdb782 --- /dev/null +++ b/src/test/java/org/gcube/dataharvest/dao/DaoTests.java @@ -0,0 +1,54 @@ +package org.gcube.dataharvest.dao; + +import java.util.LinkedHashMap; +import java.util.Properties; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.gcube.common.scope.impl.ScopeBean; +import org.gcube.dataharvest.AccountingDataHarvesterPlugin; +import org.gcube.dataharvest.DataHarvestPluginDeclaration; +import org.gcube.dataharvest.utils.ContextTest; +import org.gcube.resourcemanagement.support.server.managers.context.ContextManager; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DaoTests extends ContextTest { + + private static Logger logger = LoggerFactory.getLogger(DaoTests.class); + + public static SortedSet getContexts() throws Exception{ + SortedSet contexts = new TreeSet<>(); + LinkedHashMap map = ContextManager.readContexts(); + for(String scope : map.keySet()) { + try { + String context = map.get(scope).toString(); + contexts.add(context); + }catch (Exception e) { + throw e; + } + } + return contexts; + } + + + @Test + public void testInsertMissingContext() throws Exception { + AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(new DataHarvestPluginDeclaration()); + Properties properties = accountingDataHarvesterPlugin.getConfigParameters(); + AccountingDataHarvesterPlugin.getProperties().set(properties); + + DatabaseManager dbaseManager = new DatabaseManager(); + Dao dao = dbaseManager.dbConnect(); + + String[] contexts = new String[]{"/d4science.research-infrastructures.eu", "/d4science.research-infrastructures.eu/gCubeApps/rScience", "/d4science.research-infrastructures.eu/gCubeApps"}; + + for(String contextFullname : contexts) { + int id = dao.getOrInsertContextId(contextFullname); + logger.debug("{} is is {}", contextFullname, id); + } + + } + +}