From b15ba30b1dbc7a444bda9ab60b5e1117dbb30f67 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Mon, 21 May 2018 13:52:19 +0000 Subject: [PATCH] Code redesign Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167644 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../AccountingDataHarvesterPlugin.java | 6 +- .../DataMethodDownloadHarvester.java | 5 +- .../sobigdata/ResourceCatalogueHarvester.java | 5 +- .../sobigdata/SoBigDataHarvester.java | 15 +- .../AccountingDataHarvesterPluginTest.java | 10 +- .../java/org/gcube/dataharvest/Harvester.java | 230 ------------------ 6 files changed, 20 insertions(+), 251 deletions(-) delete mode 100644 src/test/java/org/gcube/dataharvest/Harvester.java diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index df6ed70..3568be9 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -160,8 +160,7 @@ public class AccountingDataHarvesterPlugin extends Plugin harvested = resourceCatalogueHarvester.getData(); data.addAll(harvested); } catch(Exception e) { @@ -170,8 +169,7 @@ public class AccountingDataHarvesterPlugin extends Plugin harvested = dataMethodDownloadHarvester.getData(); data.addAll(harvested); } catch(Exception e) { diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java index 2a5b471..8f806ab 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.SortedSet; import org.apache.commons.lang.Validate; import org.gcube.common.homelibrary.home.Home; @@ -32,8 +33,8 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { private int count = 0; - public DataMethodDownloadHarvester(Date start, Date end) throws ParseException { - super(start, end); + public DataMethodDownloadHarvester(Date start, Date end, SortedSet contexts) throws ParseException { + super(start, end, contexts); } @Override diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java index 4649c90..6bed1d9 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.SortedSet; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.utils.DateUtils; @@ -31,8 +32,8 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { protected String solrBaseUrl; - public ResourceCatalogueHarvester(Date start, Date end) throws ParseException { - super(start, end); + public ResourceCatalogueHarvester(Date start, Date end, SortedSet contexts) throws ParseException { + super(start, end, contexts); } protected String getSolrBaseUrl() { diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java index 7869e7f..4a3baf9 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/SoBigDataHarvester.java @@ -26,21 +26,18 @@ public abstract class SoBigDataHarvester extends BasicHarvester { protected SortedSet excludedContexts; protected SortedSet contexts; - public SoBigDataHarvester(Date start, Date end) throws ParseException { + public SoBigDataHarvester(Date start, Date end, SortedSet contexts) throws ParseException { super(start, end); - } - - public SortedSet getContexts() { - return contexts; - } - - public void setContexts(SortedSet contexts) { this.excludedContexts = getExcludedContexts(); // Adding trailing slash to SO_BIG_DATA_CONTEXT to avoid to get VO this.contexts = getSoBigDataContexts(contexts, SO_BIG_DATA_CONTEXT + "/"); logger.trace("Valid contexts are {}", contexts); } + public SortedSet getFilteredContexts() { + return contexts; + } + public static String SECONDARY_TYPE_FORMAT = "$resource/Profile/SecondaryType/text() eq '%1s'"; public static String NAME_FORMAT = "$resource/Profile/Name/text() eq '%1s'"; @@ -64,7 +61,7 @@ public abstract class SoBigDataHarvester extends BasicHarvester { return res.get(0); } - public SortedSet getExcludedContexts() { + protected SortedSet getExcludedContexts() { SortedSet excludedContexts = new TreeSet<>(); GenericResource genericResource = getGenericResource(); diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index 02d85da..7cca605 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -109,7 +109,10 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { Date start = DateUtils.getPreviousPeriod(measureType).getTime(); Date end = DateUtils.getEndDateFromStartDate(measureType, start, 1); - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); + ContextAuthorization contextAuthorization = new ContextAuthorization(); + SortedSet contexts = contextAuthorization.getContexts(); + + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); List harvestedData = resourceCatalogueHarvester.getData(); logger.debug("{}", harvestedData); @@ -133,9 +136,8 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { ContextAuthorization contextAuthorization = new ContextAuthorization(); SortedSet contexts = contextAuthorization.getContexts(); - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); - resourceCatalogueHarvester.setContexts(contexts); - SortedSet validContexts = resourceCatalogueHarvester.getContexts(); + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); + SortedSet validContexts = resourceCatalogueHarvester.getFilteredContexts(); logger.info("{}", validContexts); diff --git a/src/test/java/org/gcube/dataharvest/Harvester.java b/src/test/java/org/gcube/dataharvest/Harvester.java deleted file mode 100644 index 3a910f7..0000000 --- a/src/test/java/org/gcube/dataharvest/Harvester.java +++ /dev/null @@ -1,230 +0,0 @@ -package org.gcube.dataharvest; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import org.gcube.dataharvest.dao.Dao; -import org.gcube.dataharvest.dao.DaoException; -import org.gcube.dataharvest.dao.DatabaseConnectionData; -import org.gcube.dataharvest.dao.DatabaseParameterRetriever; -import org.gcube.dataharvest.datamodel.HarvestedData; -import org.gcube.dataharvest.harvester.BasicHarvester; -import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; -import org.gcube.dataharvest.harvester.VREUsersHarvester; -import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; -import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; -import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author Eric Perrone (ISTI - CNR) - * @author Luca Frosini (ISTI - CNR) - */ -public class Harvester { - - private static Logger logger = LoggerFactory.getLogger(Harvester.class); - - public static final String PARAMETER_FROM = "from"; - public static final String PARAMETER_TO = "to"; - public static final String TEST = "test"; - private boolean testMode = false; - private Date dateFrom, dateTo; - - public static void main(String[] args) { - Harvester harvester = new Harvester(); - try { - harvester.processParameterArray(args); - SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(harvester.getDateFrom(), harvester.getDateTo()); - harvester.runOne(socialHarvester); - // harvester.run(); - //ArrayList list = harvester.getSubTree(17); - //harvester.createSocialReports(list); - System.out.println("End."); - - } catch(Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - public Harvester() { - testMode = false; - } - - public void setDateFrom(Date dateFrom) { - this.dateFrom = dateFrom; - } - - public void setDateTo(Date dateTo) { - this.dateTo = dateTo; - } - - public void setTestMode(boolean testMode) { - this.testMode = testMode; - } - - public Date getDateFrom() { - return dateFrom; - } - - public Date getDateTo() { - return dateTo; - } - - public void runOne(BasicHarvester harvester) { - try { - List data = harvester.getData(); - if(data != null) { - insertMonthlyData((Date) dateFrom, (Date) dateTo, data); - } - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - } - - public void runAll() { - try { - // collecting info on VRE users - VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(dateFrom, dateTo); - List users = vreUsersHarvester.getData(); - insertMonthlyData((Date) dateFrom, (Date) dateTo, users); - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - - try { - // collecting info on Res. Catalogue (Dataset, Application, - // Deliverables, Methods) - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo); - List res = resourceCatalogueHarvester.getData(); - insertMonthlyData((Date) dateFrom, (Date) dateTo, res); - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - - try { - // collecting info on Data/Method download - DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo); - List res = dataMethodDownloadHarvester.getData(); - insertMonthlyData((Date) dateFrom, (Date) dateTo, res); - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - - try { - // collecting info on social (posts, replies and likes) - SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(dateFrom, dateTo); - List res = socialHarvester.getData(); - insertMonthlyData((Date) dateFrom, (Date) dateTo, res); - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - - try { - // collecting info on method invocation - TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(dateFrom, dateTo); - List res = methodInvocationHarvester.getData(); - logger.debug("{}", res); - // insertMonthlyData((Date) dateFrom, (Date) dateTo, res); - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - } - - private void insertMonthlyData(Date from, Date to, List data) { - Dao dao = null; - try { - dao = dbConnect(); - dao.insertMonthlyMeasure(data, from, to, false); - } catch(Exception e) { - logger.error(e.getLocalizedMessage()); - } finally { - if(dao != null) { - try { - dao.disconnect(); - } catch(DaoException e) { - logger.error(e.getLocalizedMessage()); - } - } - } - } - - private Dao dbConnect() throws DaoException { - DatabaseParameterRetriever dde = new DatabaseParameterRetriever(); - //dde.setTestMode(testMode); - DatabaseConnectionData dcd = dde.retrieveDatabaseInfo(); - Dao dao = new Dao(); - dao.init(); - dao.connect(dcd.getURI(), dcd.getUser(), dcd.getPassword()); - return dao; - } - - private void processParameterArray(String[] args) throws Exception { - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - for(int i = 0; i < args.length; i++) { - if(args[i].startsWith(PARAMETER_FROM)) { - String from = getArgValue(args[i]); - dateFrom = formatter.parse(from); - System.out.println(dateFrom.toString()); - } else if(args[i].startsWith(PARAMETER_TO)) { - String to = getArgValue(args[i]); - dateTo = formatter.parse(to); - System.out.println(dateTo.toString()); - } else if(args[i].toLowerCase().startsWith(TEST)) { - testMode = true; - System.out.println("Test: " + testMode); - } - - } - } - - private String getArgValue(String arg) throws Exception { - String[] tokens = arg.split("="); - if(tokens.length != 2) - throw new Exception("Argument must be in the format 'name=value'."); - return tokens[1].trim(); - } - - protected ArrayList getSubTree(int root) { - Dao dao = null; - try { - dao = dbConnect(); - ArrayList subTree = dao.getSubTree(root); - return subTree; - } catch(Exception e) { - logger.error(e.getLocalizedMessage()); - return null; - } finally { - if(dao != null) { - try { - dao.disconnect(); - } catch(DaoException e) { - logger.error(e.getLocalizedMessage()); - } - } - } - } - - protected void createSocialReports(ArrayList ids) { - Dao dao = null; - try { - dao = dbConnect(); - for(Integer contextId : ids) - dao.createSocialReport(contextId, 2018); - } catch(Exception e) { - logger.error(e.getLocalizedMessage()); - } finally { - if(dao != null) { - try { - dao.disconnect(); - } catch(DaoException e) { - logger.error(e.getLocalizedMessage()); - } - } - } - } - -}