Code redesign

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167644 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java

@@ -160,8 +160,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 				
 				try {
 					// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
-					ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
+					ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end,contexts);
-					resourceCatalogueHarvester.setContexts(contexts);
 					List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
 					data.addAll(harvested);
 				} catch(Exception e) {
@@ -170,8 +169,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
 				
 				try {
 					// Collecting info on Data/Method download
-					DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
+					DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end,contexts);;
-					dataMethodDownloadHarvester.setContexts(contexts);
 					List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
 					data.addAll(harvested);
 				} catch(Exception e) {

src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java

@@ -5,6 +5,7 @@ import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
+import java.util.SortedSet;
 
 import org.apache.commons.lang.Validate;
 import org.gcube.common.homelibrary.home.Home;
@@ -32,8 +33,8 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester {
 	
 	private int count = 0;
 	
-	public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
+	public DataMethodDownloadHarvester(Date start, Date end, SortedSet<String> contexts) throws ParseException {
-		super(start, end);
+		super(start, end, contexts);
 	}
 	
 	@Override

src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java

@@ -5,6 +5,7 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
+import java.util.SortedSet;
 
 import org.gcube.dataharvest.datamodel.HarvestedData;
 import org.gcube.dataharvest.utils.DateUtils;
@@ -31,8 +32,8 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester {
 	
 	protected String solrBaseUrl;
 	
-	public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
+	public ResourceCatalogueHarvester(Date start, Date end, SortedSet<String> contexts) throws ParseException {
-		super(start, end);
+		super(start, end, contexts);
 	}
 	
 	protected String getSolrBaseUrl() {

src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java

@@ -26,21 +26,18 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
 	protected SortedSet<String> excludedContexts;
 	protected SortedSet<String> contexts;
 	
-	public SoBigDataHarvester(Date start, Date end) throws ParseException {
+	public SoBigDataHarvester(Date start, Date end, SortedSet<String> contexts) throws ParseException {
 		super(start, end);
-	}
-	
-	public SortedSet<String> getContexts() {
-		return contexts;
-	}
-	
-	public void setContexts(SortedSet<String> contexts) {
 		this.excludedContexts = getExcludedContexts();
 		// Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO
 		this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/");
 		logger.trace("Valid contexts are {}", contexts);
 	}
 	
+	public SortedSet<String> getFilteredContexts() {
+		return contexts;
+	}
+	
 	public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'";
 	public static String NAME_FORMAT = "$resource/Profile/Name/text() eq '%1s'";
 	
@@ -64,7 +61,7 @@ public abstract class SoBigDataHarvester extends BasicHarvester {
 		return res.get(0);
 	}
 	
-	public SortedSet<String> getExcludedContexts() {
+	protected SortedSet<String> getExcludedContexts() {
 		SortedSet<String> excludedContexts = new TreeSet<>();
 		
 		GenericResource genericResource = getGenericResource();

src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java

@@ -109,7 +109,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			Date start = DateUtils.getPreviousPeriod(measureType).getTime();
 			Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1);
 			
-			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
+			ContextAuthorization contextAuthorization = new ContextAuthorization();
+			SortedSet<String> contexts = contextAuthorization.getContexts();
+			
+			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
 			List<HarvestedData> harvestedData = resourceCatalogueHarvester.getData();
 			
 			logger.debug("{}", harvestedData);
@@ -133,9 +136,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
 			ContextAuthorization contextAuthorization = new ContextAuthorization();
 			SortedSet<String> contexts = contextAuthorization.getContexts();
 			
-			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
+			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts);
-			resourceCatalogueHarvester.setContexts(contexts);
+			SortedSet<String> validContexts = resourceCatalogueHarvester.getFilteredContexts();
-			SortedSet<String> validContexts = resourceCatalogueHarvester.getContexts();
 			
 			logger.info("{}", validContexts);
 			

src/test/java/org/gcube/dataharvest/Harvester.java

@@ -1,230 +0,0 @@
-package org.gcube.dataharvest;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.gcube.dataharvest.dao.Dao;
-import org.gcube.dataharvest.dao.DaoException;
-import org.gcube.dataharvest.dao.DatabaseConnectionData;
-import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
-import org.gcube.dataharvest.datamodel.HarvestedData;
-import org.gcube.dataharvest.harvester.BasicHarvester;
-import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
-import org.gcube.dataharvest.harvester.VREUsersHarvester;
-import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
-import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
-import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author Eric Perrone (ISTI - CNR)
- * @author Luca Frosini (ISTI - CNR)
- */
-public class Harvester {
-	
-	private static Logger logger = LoggerFactory.getLogger(Harvester.class);
-	
-	public static final String PARAMETER_FROM = "from";
-	public static final String PARAMETER_TO = "to";
-	public static final String TEST = "test";
-	private boolean testMode = false;
-	private Date dateFrom, dateTo;
-	
-	public static void main(String[] args) {
-		Harvester harvester = new Harvester();
-		try {
-			harvester.processParameterArray(args);
-			SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(harvester.getDateFrom(), harvester.getDateTo());
-			harvester.runOne(socialHarvester);
-			// harvester.run();
-			//ArrayList<Integer> list = harvester.getSubTree(17);
-			//harvester.createSocialReports(list);
-			System.out.println("End.");
-			
-		} catch(Exception e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-	}
-	
-	public Harvester() {
-		testMode = false;
-	}
-	
-	public void setDateFrom(Date dateFrom) {
-		this.dateFrom = dateFrom;
-	}
-	
-	public void setDateTo(Date dateTo) {
-		this.dateTo = dateTo;
-	}
-	
-	public void setTestMode(boolean testMode) {
-		this.testMode = testMode;
-	}
-	
-	public Date getDateFrom() {
-		return dateFrom;
-	}
-	
-	public Date getDateTo() {
-		return dateTo;
-	}
-	
-	public void runOne(BasicHarvester harvester) {
-		try {
-			List<HarvestedData> data = harvester.getData();
-			if(data != null) {
-				insertMonthlyData((Date) dateFrom, (Date) dateTo, data);
-			}
-		} catch(Exception x) {
-			logger.error(x.getLocalizedMessage());
-		}
-	}
-	
-	public void runAll() {
-		try {
-			// collecting info on VRE users
-			VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(dateFrom, dateTo);
-			List<HarvestedData> users = vreUsersHarvester.getData();
-			insertMonthlyData((Date) dateFrom, (Date) dateTo, users);
-		} catch(Exception x) {
-			logger.error(x.getLocalizedMessage());
-		}
-		
-		try {
-			// collecting info on Res. Catalogue (Dataset, Application,
-			// Deliverables, Methods)
-			ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
-			List<HarvestedData> res = resourceCatalogueHarvester.getData();
-			insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
-		} catch(Exception x) {
-			logger.error(x.getLocalizedMessage());
-		}
-		
-		try {
-			// collecting info on Data/Method download
-			DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
-			List<HarvestedData> res = dataMethodDownloadHarvester.getData();
-			insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
-		} catch(Exception x) {
-			logger.error(x.getLocalizedMessage());
-		}
-		
-		try {
-			// collecting info on social (posts, replies and likes)
-			SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(dateFrom, dateTo);
-			List<HarvestedData> res = socialHarvester.getData();
-			insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
-		} catch(Exception x) {
-			logger.error(x.getLocalizedMessage());
-		}
-		
-		try {
-			// collecting info on method invocation
-			TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(dateFrom, dateTo);
-			List<HarvestedData> res = methodInvocationHarvester.getData();
-			logger.debug("{}", res);
-			// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
-		} catch(Exception x) {
-			logger.error(x.getLocalizedMessage());
-		}
-	}
-	
-	private void insertMonthlyData(Date from, Date to, List<HarvestedData> data) {
-		Dao dao = null;
-		try {
-			dao = dbConnect();
-			dao.insertMonthlyMeasure(data, from, to, false);
-		} catch(Exception e) {
-			logger.error(e.getLocalizedMessage());
-		} finally {
-			if(dao != null) {
-				try {
-					dao.disconnect();
-				} catch(DaoException e) {
-					logger.error(e.getLocalizedMessage());
-				}
-			}
-		}
-	}
-	
-	private Dao dbConnect() throws DaoException {
-		DatabaseParameterRetriever dde = new DatabaseParameterRetriever();
-		//dde.setTestMode(testMode);
-		DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
-		Dao dao = new Dao();
-		dao.init();
-		dao.connect(dcd.getURI(), dcd.getUser(), dcd.getPassword());
-		return dao;
-	}
-	
-	private void processParameterArray(String[] args) throws Exception {
-		SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-		for(int i = 0; i < args.length; i++) {
-			if(args[i].startsWith(PARAMETER_FROM)) {
-				String from = getArgValue(args[i]);
-				dateFrom = formatter.parse(from);
-				System.out.println(dateFrom.toString());
-			} else if(args[i].startsWith(PARAMETER_TO)) {
-				String to = getArgValue(args[i]);
-				dateTo = formatter.parse(to);
-				System.out.println(dateTo.toString());
-			} else if(args[i].toLowerCase().startsWith(TEST)) {
-				testMode = true;
-				System.out.println("Test: " + testMode);
-			}
-			
-		}
-	}
-	
-	private String getArgValue(String arg) throws Exception {
-		String[] tokens = arg.split("=");
-		if(tokens.length != 2)
-			throw new Exception("Argument must be in the format 'name=value'.");
-		return tokens[1].trim();
-	}
-	
-	protected ArrayList<Integer> getSubTree(int root) {
-		Dao dao = null;
-		try {
-			dao = dbConnect();
-			ArrayList<Integer> subTree = dao.getSubTree(root);
-			return subTree;
-		} catch(Exception e) {
-			logger.error(e.getLocalizedMessage());
-			return null;
-		} finally {
-			if(dao != null) {
-				try {
-					dao.disconnect();
-				} catch(DaoException e) {
-					logger.error(e.getLocalizedMessage());
-				}
-			}
-		}
-	}
-	
-	protected void createSocialReports(ArrayList<Integer> ids) {
-		Dao dao = null;
-		try {
-			dao = dbConnect();
-			for(Integer contextId : ids)
-				dao.createSocialReport(contextId, 2018);
-		} catch(Exception e) {
-			logger.error(e.getLocalizedMessage());
-		} finally {
-			if(dao != null) {
-				try {
-					dao.disconnect();
-				} catch(DaoException e) {
-					logger.error(e.getLocalizedMessage());
-				}
-			}
-		}
-	}
-	
-}