Added auto creation of contexts

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167822 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java

@@ -64,19 +64,19 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 		return properties;
 	}
 
-	public void getConfigParameters() throws IOException {
-		try {
+	public Properties getConfigParameters() throws IOException {
 		Properties properties = new Properties();
+		try {
 			InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader()
 					.getResourceAsStream(PROPERTY_FILENAME);
 			properties.load(input);
-			getProperties().set(properties);
+			return properties;
 		} catch(Exception e) {
 			logger.warn(
 					"Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults",
 					PROPERTY_FILENAME);
 		}
-
+		return properties;
 	}
 
 	/** {@inheritDoc} */
@@ -124,7 +124,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 		logger.debug("Harvesting from {} to {}", DateUtils.format(start),
 				DateUtils.format(end));
 
-		getConfigParameters();
+		Properties properties = getConfigParameters();
+		getProperties().set(properties);
 		
 		ContextAuthorization contextAuthorization = new ContextAuthorization();
 

src/main/java/org/gcube/dataharvest/dao/Dao.java

@@ -7,8 +7,10 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.List;
 
+import org.gcube.common.scope.impl.ScopeBean;
 import org.gcube.dataharvest.datamodel.HarvestedData;
 import org.gcube.dataharvest.utils.DateUtils;
 import org.postgresql.util.PSQLException;
@@ -140,7 +142,7 @@ public class Dao {
 	 * @param to
 	 * @throws DaoException
 	 */
-	public void insertMonthlyMeasure(List<HarvestedData> data, java.util.Date from, java.util.Date to, boolean doUpdate)
+	public void insertMonthlyMeasure(List<HarvestedData> data, Date from, Date to, boolean doUpdate)
 			throws DaoException {
 		// first of all: check if data of the same type are already in the
 		// database.
@@ -166,10 +168,11 @@ public class Dao {
 		monthFrom++; // because january = 0...
 		
 		try {
-			for (HarvestedData harvest : data) {
-				String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType()
-						+ " and context_id=(select id from context where dname='" + harvest.getContext()
-						+ "') and month=" + monthFrom + " and year=" + yearFrom;
+			for(HarvestedData harvestedData : data) {
+				int contextId = getOrInsertContextId(harvestedData.getContext());
+				
+				String query = "select id from monthly_measure where measure_type_id=" + harvestedData.getDataType()
+						+ " and context_id=" + contextId + " and month=" + monthFrom + " and year=" + yearFrom;
 				logger.debug(query);
 				Statement sel = conn.createStatement();
 				ResultSet rs = sel.executeQuery(query);
@@ -178,21 +181,21 @@ public class Dao {
 						// record found: update it
 						Statement s = conn.createStatement();
 						int id = rs.getInt("id");
-						String update = "update monthly_measure set measure=" + harvest.getMeasure() + " where id="
+						String update = "update monthly_measure set measure=" + harvestedData.getMeasure() + " where id="
 								+ id;
 						logger.debug(update);
 						s.execute(update);
 						s.close();
 					} else {
-						logger.warn("Skipped " + harvest.getContext());
+						logger.warn("Skipped " + harvestedData.getContext());
 					}
 				} else {
 					// record not found: insert new record
 					Statement s = conn.createStatement();
 					String insert = "insert into monthly_measure (year, month, measure, measure_type_id, context_id, day) values (";
-					insert += yearFrom + "," + monthFrom + "," + harvest.getMeasure() + "," + harvest.getDataType()
+					insert += yearFrom + "," + monthFrom + "," + harvestedData.getMeasure() + "," + harvestedData.getDataType()
 							+ ",";
-					insert += "(select id from context where dname='" + harvest.getContext() + "'),";
+					insert += "(select id from context where dname='" + harvestedData.getContext() + "'),";
 					insert += "'" + yearFrom + "-" + monthFrom + "-01'";
 					insert += ")";
 					logger.debug(insert);
@@ -214,6 +217,63 @@ public class Dao {
 		}
 	}
 	
+	public enum contextType {
+		INFRASTRUCTURE, VO, VRE
+	}
+	
+	public int getOrInsertContextId(String contextFullName) throws SQLException {
+		String query = "select id from context where dname='" + contextFullName + "'";
+		logger.debug(query);
+		Statement sel = conn.createStatement();
+		ResultSet rs = sel.executeQuery(query);
+		if(rs.next()) {
+			// context found
+			int id = rs.getInt("id");
+			logger.debug("Context {} has id {}", contextFullName, id);
+			return id;
+		} else {
+			// context not found: insert new record
+			int parentId = 0; // The id of D4Science infrastructure as aggregator
+			int lastIndexOfSlash = contextFullName.lastIndexOf("/");
+			if(lastIndexOfSlash!=0) {
+				String parentContextFullName = contextFullName.substring(0, contextFullName.lastIndexOf("/"));
+				parentId = getOrInsertContextId(parentContextFullName);
+			}
+			
+			// It is a new context and we don't know which context type is. Using 0 for ROOT, 1 for VO, 2 for VRE
+			int contextTypeId = contextType.INFRASTRUCTURE.ordinal();
+			ScopeBean scopeBean = new ScopeBean(contextFullName);
+			switch(scopeBean.type()) {
+				case INFRASTRUCTURE:
+					contextTypeId = contextType.INFRASTRUCTURE.ordinal();
+					break;
+				case VO:
+					contextTypeId = contextType.VO.ordinal();
+					break;
+				case VRE:
+					contextTypeId = contextType.VRE.ordinal();
+					break;
+				default:
+					break;
+			}
+			
+			Statement s = conn.createStatement();
+			String insert = "insert into context (name, context_id, context_type_id, dname) values ('";
+			insert += contextFullName + "'," + parentId + "," + contextTypeId + ", '" + contextFullName;
+			insert += "')";
+			logger.debug(insert);
+			
+			s.execute(insert);
+			s.close();
+			
+		}
+		
+		rs.close();
+		sel.close();
+		
+		return getOrInsertContextId(contextFullName);
+	}
+	
 	public ArrayList<Integer> getSubTree(Integer rootId) throws DaoException {
 		String queryBase = "select id from context where context_id in (%%)";
 		ArrayList<Integer> subTree = new ArrayList<>();
@@ -244,8 +304,7 @@ public class Dao {
 						subTree.add(dbId);
 						temp.add(dbId);
 					}
-				}
-				else {
+				} else {
 					again = false;
 				}
 				rs.close();
@@ -296,7 +355,6 @@ public class Dao {
 		}
 	}
 	
-
 	/**
 	 * Dummy tester
 	 * 

src/main/java/org/gcube/dataharvest/dao/DatabaseParameterRetriever.java

@@ -56,6 +56,7 @@ public class DatabaseParameterRetriever {
 		String password = "";
 		
 		if(localDB) {
+			logger.debug("Using configuration from local config");
 			uri = properties.getProperty(DB_URI);
 			username = properties.getProperty(DB_USERNAME);
 			password = properties.getProperty(DB_PASSWORD);
@@ -72,8 +73,12 @@ public class DatabaseParameterRetriever {
 				
 				DiscoveryClient<ServiceEndpoint> client = ICFactory.clientFor(ServiceEndpoint.class);
 				List<ServiceEndpoint> serviceEndpoints = client.submit(query);
+				if(serviceEndpoints.size()==0) {
+					throw new DaoException("No endpoints found to get database connection.");
+				}
+				
 				if(serviceEndpoints.size() > 1) {
-					throw new DaoException("More than one endpoint found. Not sure which one use.");
+					throw new DaoException("More than one endpoint found to get database connection. Not sure which one use.");
 				}
 				
 				Group<AccessPoint> accessPoints = serviceEndpoints.get(0).profile().accessPoints();

src/test/java/org/gcube/dataharvest/dao/DaoTests.java

@@ -0,0 +1,54 @@
+package org.gcube.dataharvest.dao;
+
+import java.util.LinkedHashMap;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.gcube.common.scope.impl.ScopeBean;
+import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
+import org.gcube.dataharvest.DataHarvestPluginDeclaration;
+import org.gcube.dataharvest.utils.ContextTest;
+import org.gcube.resourcemanagement.support.server.managers.context.ContextManager;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DaoTests  extends ContextTest {
+	
+	private static Logger logger = LoggerFactory.getLogger(DaoTests.class);
+
+	public static SortedSet<String> getContexts() throws Exception{
+		SortedSet<String> contexts = new TreeSet<>();
+		LinkedHashMap<String,ScopeBean> map = ContextManager.readContexts();
+		for(String scope : map.keySet()) {
+			try {
+				String context = map.get(scope).toString();
+				contexts.add(context);
+			}catch (Exception e) {
+				throw e;
+			}
+		}
+		return contexts;
+	}
+	
+	
+	@Test
+	public void testInsertMissingContext() throws Exception {
+		AccountingDataHarvesterPlugin accountingDataHarvesterPlugin = new AccountingDataHarvesterPlugin(new DataHarvestPluginDeclaration());
+		Properties properties = accountingDataHarvesterPlugin.getConfigParameters();
+		AccountingDataHarvesterPlugin.getProperties().set(properties);
+		
+		DatabaseManager dbaseManager = new DatabaseManager();
+		Dao dao = dbaseManager.dbConnect();
+		
+		String[] contexts = new String[]{"/d4science.research-infrastructures.eu", "/d4science.research-infrastructures.eu/gCubeApps/rScience", "/d4science.research-infrastructures.eu/gCubeApps"};
+		
+		for(String contextFullname : contexts) {
+			int id = dao.getOrInsertContextId(contextFullname);
+			logger.debug("{} is is {}", contextFullname, id);
+		}
+		
+	}
+	
+}