Added auto creation of contexts

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167822 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java

@@ -64,19 +64,19 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 		return properties;
 	}
 
-	public void getConfigParameters() throws IOException {
+	public Properties getConfigParameters() throws IOException {
+		Properties properties = new Properties();
 		try {
-			Properties properties = new Properties();
 			InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader()
 					.getResourceAsStream(PROPERTY_FILENAME);
 			properties.load(input);
-			getProperties().set(properties);
+			return properties;
 		} catch(Exception e) {
 			logger.warn(
 					"Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults",
 					PROPERTY_FILENAME);
 		}
-
+		return properties;
 	}
 
 	/** {@inheritDoc} */
@@ -124,8 +124,9 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 		logger.debug("Harvesting from {} to {}", DateUtils.format(start),
 				DateUtils.format(end));
 
-		getConfigParameters();
-
+		Properties properties = getConfigParameters();
+		getProperties().set(properties);
+		
 		ContextAuthorization contextAuthorization = new ContextAuthorization();
 
 		DatabaseManager dbaseManager = new DatabaseManager();

src/main/java/org/gcube/dataharvest/dao/Dao.java

@@ -7,8 +7,10 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.List;
 
+import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.dataharvest.datamodel.HarvestedData;
 import org.gcube.dataharvest.utils.DateUtils;
 import org.postgresql.util.PSQLException;
@@ -22,7 +24,7 @@ import org.slf4j.LoggerFactory;
 public class Dao {
 	private Connection conn = null;
 	private static Logger logger = LoggerFactory.getLogger(Dao.class);
-
+	
 	/**
 	 * Class constructor. This method must be called before any other class
 	 * method.
@@ -32,12 +34,12 @@ public class Dao {
 	public void init() throws DaoException {
 		try {
 			Class.forName("org.postgresql.Driver");
-		} catch (ClassNotFoundException ex) {
+		} catch(ClassNotFoundException ex) {
 			logger.error(ex.getLocalizedMessage());
 			throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
 		}
 	}
-
+	
 	/**
 	 * Connect a database
 	 * 
@@ -50,12 +52,12 @@ public class Dao {
 		try {
 			conn = DriverManager.getConnection(url, user, password);
 			logger.debug("Connected to: " + url);
-		} catch (SQLException ex) {
+		} catch(SQLException ex) {
 			logger.error(ex.getLocalizedMessage());
 			throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
 		}
 	}
-
+	
 	/**
 	 * Release a database connection
 	 *
@@ -63,17 +65,17 @@ public class Dao {
 	 */
 	public void disconnect() throws DaoException {
 		try {
-			if (conn != null) {
+			if(conn != null) {
 				conn.close();
 				conn = null;
 				logger.debug("Disconnecting from database");
 			}
-		} catch (SQLException ex) {
+		} catch(SQLException ex) {
 			logger.error(ex.getLocalizedMessage());
 			throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
 		}
 	}
-
+	
 	/**
 	 * Check the connection with the database.
 	 * 
@@ -82,7 +84,7 @@ public class Dao {
 	public boolean isConnected() {
 		return (!(conn == null));
 	}
-
+	
 	/**
 	 * Getter for the database connection
 	 * 
@@ -91,7 +93,7 @@ public class Dao {
 	public Connection getConnection() {
 		return conn;
 	}
-
+	
 	/**
 	 * Read on the database all active VRES
 	 * 
@@ -106,32 +108,32 @@ public class Dao {
 		try {
 			s = conn.createStatement();
 			rs = s.executeQuery(query);
-			while (rs.next()) {
+			while(rs.next()) {
 				list.add(rs.getString("dname"));
 			}
-
+			
 			return list.toArray(new String[list.size()]);
-		} catch (Exception ex) {
+		} catch(Exception ex) {
 			logger.error(ex.getLocalizedMessage());
 			return null;
 		} finally {
-			if (rs != null) {
+			if(rs != null) {
 				try {
 					rs.close();
-				} catch (SQLException ex) {
+				} catch(SQLException ex) {
 					// do nothing
 				}
 			}
-			if (s != null) {
+			if(s != null) {
 				try {
 					s.close();
-				} catch (SQLException ex) {
+				} catch(SQLException ex) {
 					// do nothing
 				}
 			}
 		}
 	}
-
+	
 	/**
 	 * This method insert/update data in the monthly_measure database table.
 	 * 
@@ -140,7 +142,7 @@ public class Dao {
 	 * @param to
 	 * @throws DaoException
 	 */
-	public void insertMonthlyMeasure(List<HarvestedData> data, java.util.Date from, java.util.Date to, boolean doUpdate)
+	public void insertMonthlyMeasure(List<HarvestedData> data, Date from, Date to, boolean doUpdate)
 			throws DaoException {
 		// first of all: check if data of the same type are already in the
 		// database.
@@ -151,69 +153,127 @@ public class Dao {
 		int yearFrom = cFrom.get(Calendar.YEAR);
 		int monthTo = cTo.get(Calendar.MONTH);
 		int yearTo = cTo.get(Calendar.YEAR);
-
-		if ((monthFrom != monthTo) || (yearFrom != yearTo)) {
+		
+		if((monthFrom != monthTo) || (yearFrom != yearTo)) {
 			String err = "Invalid time period. The time period MUST refer one month.";
 			logger.error(err);
 			throw new DaoException(err, null);
 		}
-
-		if (data == null || data.size() <= 0) {
+		
+		if(data == null || data.size() <= 0) {
 			String err = "No data passed in input. Aborting operation.";
 			logger.error(err);
 			throw new DaoException(err, null);
 		}
 		monthFrom++; // because january = 0...
-
+		
 		try {
-			for (HarvestedData harvest : data) {
-				String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType()
-						+ " and context_id=(select id from context where dname='" + harvest.getContext()
-						+ "') and month=" + monthFrom + " and year=" + yearFrom;
+			for(HarvestedData harvestedData : data) {
+				int contextId = getOrInsertContextId(harvestedData.getContext());
+				
+				String query = "select id from monthly_measure where measure_type_id=" + harvestedData.getDataType()
+						+ " and context_id=" + contextId + " and month=" + monthFrom + " and year=" + yearFrom;
 				logger.debug(query);
 				Statement sel = conn.createStatement();
 				ResultSet rs = sel.executeQuery(query);
-				if (rs.next()) {
-					if (doUpdate) {
+				if(rs.next()) {
+					if(doUpdate) {
 						// record found: update it
 						Statement s = conn.createStatement();
 						int id = rs.getInt("id");
-						String update = "update monthly_measure set measure=" + harvest.getMeasure() + " where id="
+						String update = "update monthly_measure set measure=" + harvestedData.getMeasure() + " where id="
 								+ id;
 						logger.debug(update);
 						s.execute(update);
 						s.close();
 					} else {
-						logger.warn("Skipped " + harvest.getContext());
+						logger.warn("Skipped " + harvestedData.getContext());
 					}
 				} else {
 					// record not found: insert new record
 					Statement s = conn.createStatement();
 					String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values (";
-					insert += yearFrom + "," + monthFrom + "," + harvest.getMeasure() + "," + harvest.getDataType()
+					insert += yearFrom + "," + monthFrom + "," + harvestedData.getMeasure() + "," + harvestedData.getDataType()
 							+ ",";
-					insert += "(select id from context where dname='" + harvest.getContext() + "'),";
+					insert += "(select id from context where dname='" + harvestedData.getContext() + "'),";
 					insert += "'" + yearFrom + "-" + monthFrom + "-01'";
 					insert += ")";
 					logger.debug(insert);
 					s.execute(insert);
 					s.close();
 				}
-
+				
 				rs.close();
 				sel.close();
-
+				
 			}
-		} catch (PSQLException x) {
+		} catch(PSQLException x) {
 			// handled exception: try to iterate...
 			logger.error(x.getLocalizedMessage());
-		} catch (Exception x) {
+		} catch(Exception x) {
 			// not handled exception: stop
 			logger.error(x.getLocalizedMessage());
 			throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x);
 		}
 	}
-
+	
+	public enum contextType {
+		INFRASTRUCTURE, VO, VRE
+	}
+	
+	public int getOrInsertContextId(String contextFullName) throws SQLException {
+		String query = "select id from context where dname='" + contextFullName + "'";
+		logger.debug(query);
+		Statement sel = conn.createStatement();
+		ResultSet rs = sel.executeQuery(query);
+		if(rs.next()) {
+			// context found
+			int id = rs.getInt("id");
+			logger.debug("Context {} has id {}", contextFullName, id);
+			return id;
+		} else {
+			// context not found: insert new record
+			int parentId = 0; // The id of D4Science infrastructure as aggregator
+			int lastIndexOfSlash = contextFullName.lastIndexOf("/");
+			if(lastIndexOfSlash!=0) {
+				String parentContextFullName = contextFullName.substring(0, contextFullName.lastIndexOf("/"));
+				parentId = getOrInsertContextId(parentContextFullName);
+			}
+			
+			// It is a new context and we don't know which context type is. Using 0 for ROOT, 1 for VO, 2 for VRE
+			int contextTypeId = contextType.INFRASTRUCTURE.ordinal();
+			ScopeBean scopeBean = new ScopeBean(contextFullName);
+			switch(scopeBean.type()) {
+				case INFRASTRUCTURE:
+					contextTypeId = contextType.INFRASTRUCTURE.ordinal();
+					break;
+				case VO:
+					contextTypeId = contextType.VO.ordinal();
+					break;
+				case VRE:
+					contextTypeId = contextType.VRE.ordinal();
+					break;
+				default:
+					break;
+			}
+			
+			Statement s = conn.createStatement();
+			String insert = "insert into context (name, context_id, context_type_id, dname) values ('";
+			insert += contextFullName + "'," + parentId + "," + contextTypeId + ", '" + contextFullName;
+			insert += "')";
+			logger.debug(insert);
+			
+			s.execute(insert);
+			s.close();
+			
+		}
+		
+		rs.close();
+		sel.close();
+		
+		return getOrInsertContextId(contextFullName);
+	}
+	
 	public ArrayList<Integer> getSubTree(Integer rootId) throws DaoException {
 		String queryBase = "select id from context where context_id in (%%)";
 		ArrayList<Integer> subTree = new ArrayList<>();
@@ -223,34 +283,33 @@ public class Dao {
 		temp.add(rootId);
 		subTree.add(rootId);
 		boolean again = true;
-		for (int i = 0; (i < 10) && (again); i++) {
+		for(int i = 0; (i < 10) && (again); i++) {
 			try {
 				String listId = "";
-				for (Integer id : temp) {
+				for(Integer id : temp) {
 					listId += "," + id;
 				}
 				listId = listId.substring(1);
-
+				
 				String query = queryBase.replace("%%", listId);
 				s = conn.createStatement();
 				rs = s.executeQuery(query);
-				if (rs.next()) {
+				if(rs.next()) {
 					temp = new ArrayList<>();
-					Integer dbId = rs.getInt("id"); 
+					Integer dbId = rs.getInt("id");
 					subTree.add(dbId);
 					temp.add(dbId);
-					while (rs.next()) {
-						dbId = rs.getInt("id"); 
+					while(rs.next()) {
+						dbId = rs.getInt("id");
 						subTree.add(dbId);
 						temp.add(dbId);
 					}
-				}
-				else {
+				} else {
 					again = false;
-				} 
+				}
 				rs.close();
 				s.close();
-			} catch (Exception x) {
+			} catch(Exception x) {
 				logger.error(x.getLocalizedMessage());
 				throw new DaoException(x.getClass().getName() + "::" + x.getLocalizedMessage(), x);
 			}
@@ -258,7 +317,7 @@ public class Dao {
 		
 		return subTree;
 	}
-
+	
 	public void createSocialReport(int contextId, int orderBy) throws DaoException {
 		Statement rep = null;
 		ResultSet rs = null;
@@ -271,32 +330,31 @@ public class Dao {
 			rep.execute(report);
 			String query = "select id from report where name=\'VRE Social\' order by id desc";
 			rs = rep.executeQuery(query);
-			if (rs.next()) {
+			if(rs.next()) {
 				int id = rs.getInt("id");
 				String reportItem = "insert into report_item (report_id, type_id_1, type_id_1_name) ";
 				reportItem += "values (" + id + ", 8, \'mt\'), (" + id + ", 9, \'mt\'), (" + id + ", 10, \'mt\')";
 				rep.execute(reportItem);
 			} else
 				throw new DaoException("No report id.", null);
-
-		} catch (SQLException ex) {
+			
+		} catch(SQLException ex) {
 			logger.error(ex.getLocalizedMessage());
 			throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
 		} finally {
 			try {
-				if (rs != null) {
+				if(rs != null) {
 					rs.close();
 				}
-				if (rep != null) {
+				if(rep != null) {
 					rep.close();
 				}
-			} catch (SQLException ex) {
+			} catch(SQLException ex) {
 				logger.error(ex.getLocalizedMessage());
 			}
 		}
 	}
 	
-
 	/**
 	 * Dummy tester
 	 * 
@@ -312,17 +370,17 @@ public class Dao {
 		try {
 			s = conn.createStatement();
 			s.execute(insert);
-		} catch (SQLException ex) {
+		} catch(SQLException ex) {
 			logger.error(ex.getLocalizedMessage());
 			throw new DaoException(ex.getLocalizedMessage(), ex.getCause());
 		} finally {
-			if (s != null)
+			if(s != null)
 				try {
 					s.close();
-				} catch (SQLException ex) {
+				} catch(SQLException ex) {
 					logger.error(ex.getLocalizedMessage());
 				}
 		}
 	}
-
+	
 }

src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java

@@ -56,6 +56,7 @@ public class DatabaseParameterRetriever {
 		String password = "";
 		
 		if(localDB) {
+			logger.debug("Using configuration from local config");
 			uri = properties.getProperty(DB_URI);
 			username = properties.getProperty(DB_USERNAME);
 			password = properties.getProperty(DB_PASSWORD);
@@ -72,8 +73,12 @@ public class DatabaseParameterRetriever {
 				
 				DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
 				List<ServiceEndpoint> serviceEndpoints = client.submit(query);
+				if(serviceEndpoints.size()==0) {
+					throw new DaoException("No endpoints found to get database connection.");
+				}
+				
 				if(serviceEndpoints.size() > 1) {
-					throw new DaoException("More than one endpoint found. Not sure which one use.");
+					throw new DaoException("More than one endpoint found to get database connection. Not sure which one use.");
 				}
 				
 				Group<AccessPoint> accessPoints = serviceEndpoints.get(0).profile().accessPoints();

src/test/java/org/gcube/dataharvest/dao/DaoTests.java

@@ -0,0 +1,54 @@
+package org.gcube.dataharvest.dao;
+
+import java.util.LinkedHashMap;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.gcube.common.scope.impl.ScopeBean;
+import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
+import org.gcube.dataharvest.DataHarvestPluginDeclaration;
+import org.gcube.dataharvest.utils.ContextTest;
+import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DaoTests  extends ContextTest {
+	
+	private static Logger logger = LoggerFactory.getLogger(DaoTests.class);
+
+	public static SortedSet<String> getContexts() throws Exception{
+		SortedSet<String> contexts = new TreeSet<>();
+		LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
+		for(String scope : map.keySet()) {
+			try {
+				String context = map.get(scope).toString();
+				contexts.add(context);
+			}catch (Exception e) {
+				throw e;
+			}
+		}
+		return contexts;
+	}
+	
+	
+	@Test
+	public void testInsertMissingContext() throws Exception {
+		AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(new DataHarvestPluginDeclaration());
+		Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
+		AccountingDataHarvesterPlugin.getProperties().set(properties);
+		
+		DatabaseManager dbaseManager = new DatabaseManager();
+		Dao dao = dbaseManager.dbConnect();
+		
+		String[] contexts = new String[]{"/d4science.research-infrastructures.eu", "/d4science.research-infrastructures.eu/gCubeApps/rScience", "/d4science.research-infrastructures.eu/gCubeApps"};
+		
+		for(String contextFullname : contexts) {
+			int id = dao.getOrInsertContextId(contextFullname);
+			logger.debug("{} is is {}", contextFullname, id);
+		}
+		
+	}
+	
+}