From 67ae816424d2d1b72a7dccdb35370d61f1707d14 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Mon, 28 May 2018 14:53:00 +0000 Subject: [PATCH] Code redesign Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167798 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../DataMethodDownloadHarvester.java | 109 +++++++++--------- .../sobigdata/ResourceCatalogueHarvester.java | 10 +- .../AccountingDataHarvesterPluginTest.java | 4 +- 3 files changed, 62 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java index 5a33803..5ef35c3 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java @@ -32,11 +32,11 @@ import org.slf4j.LoggerFactory; * @author Francesco Mangiacrapa (ISTI - CNR) */ public class DataMethodDownloadHarvester extends SoBigDataHarvester { - + private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class); - + private int count = 0; - + /** * Instantiates a new data method download harvester. * @@ -49,78 +49,82 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { public DataMethodDownloadHarvester(Date start, Date end, SortedSet contexts) throws Exception { super(start, end, contexts); } - + /* (non-Javadoc) * @see org.gcube.dataharvest.harvester.BasicHarvester#getData() */ @Override public List getData() throws Exception { String defaultContext = Utils.getCurrentContext(); - logger.debug("The context is: "+defaultContext); - + logger.debug("The context is {}", defaultContext); + try { - + String vreName = getVRENameToHL(defaultContext); - logger.debug("Getting VRE Name to HL from context/scope returns: "+vreName); - + logger.debug("Getting VRE Name to HL from context/scope returns {} ", vreName); + String user = vreName + "-Manager"; - logger.debug("Using user '"+user+"' to getHome from HL"); - - //ISTANCING HL AND GETTING HOME FOR VRE MANAGER + logger.debug("Using user '{}' to getHome from HL", user); + + //Getting HL instance and home for VRE MANAGER HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager(); @SuppressWarnings("deprecation") Home home = manager.getHome(user); JCRWorkspace ws = (JCRWorkspace) home.getWorkspace(); -// + String path = "/Workspace/MySpecialFolders/" + vreName; - logger.debug("Getting item by Path: "+path); + logger.debug("Getting item by Path {}", path); JCRWorkspaceItem item = (JCRWorkspaceItem) ws.getItemByPath(path); -// - logger.info("Analyzing " + defaultContext + " in the period [" + start.toString() + " to " + end.toString() +"] starting from root: "+item.getName()); - - HarvestedData defaultHarvesteData = new HarvestedData(HarvestedDataKey.DATA_METHOD_DOWNLOAD, defaultContext, count); - + + logger.debug("Analyzing {} in the period [{} to {}] starting from root {}", defaultContext, + DateUtils.format(start), DateUtils.format(end), item.getName()); + + HarvestedData defaultHarvesteData = new HarvestedData(HarvestedDataKey.DATA_METHOD_DOWNLOAD, defaultContext, + count); + List data = new ArrayList(); - for (WorkspaceItem children: item.getChildren()) { + for(WorkspaceItem children : item.getChildren()) { count = 0; //resettings the counter HarvestedData harvestedData; //Getting statistics for folder - if(children.isFolder()){ - logger.info("Getting statistics for folder: "+children.getName()); + if(children.isFolder()) { + logger.debug("Getting statistics for folder {}", children.getName()); getStats(children, start, end); - - String normalizedName = children.getName().replaceAll("[^A-Za-z0-9]",""); + + String normalizedName = children.getName().replaceAll("[^A-Za-z0-9]", ""); String scope = mapWsFolderNameToVRE.get(normalizedName); - //Checking if it is a VRE name to right accouning... - if(scope!=null && !scope.isEmpty()){ - logger.info("Found scope '" + scope + "' matching with normalized VRE name: "+normalizedName); + //Checking if it is a VRE name to right accounting... + if(scope != null && !scope.isEmpty()) { + logger.debug("Found context '{}' matching with normalized VRE name {} ", scope, normalizedName); harvestedData = new HarvestedData(HarvestedDataKey.DATA_METHOD_DOWNLOAD, scope, count); data.add(harvestedData); - logger.info("Added data: "+harvestedData); - }else{ - logger.info("No scope found matching the folder name: "+normalizedName +", accounting its stats in the default context: "+defaultContext); + logger.debug("Added data {}", harvestedData); + } else { + logger.debug( + "No scope found matching the folder name {}, accounting its stats in the default context {}", + normalizedName, defaultContext); //INCREASING THE DEFAULT CONTEXT COUNTER... - defaultHarvesteData.setMeasure(defaultHarvesteData.getMeasure()+count); - logger.info("Increased default context stats: "+defaultHarvesteData); + defaultHarvesteData.setMeasure(defaultHarvesteData.getMeasure() + count); + logger.trace("Increased default context stats {}", defaultHarvesteData); } - + } } - + //ADDING DEFAULT ACCOUNTING data.add(defaultHarvesteData); - - logger.info("In the period [from {} to {} ] returning workspace accouting data {}", DateUtils.format(start), DateUtils.format(end), data); - + + logger.debug("In the period [from {} to {} ] returning workspace accouting data {}", DateUtils.format(start), + DateUtils.format(end), data); + return data; - + } catch(Exception e) { throw e; } - + } - - + /** * Gets the stats. * @@ -138,34 +142,33 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { getStats(child, start, end); } else { try { - + List accounting = baseItem.getAccounting(); for(AccountingEntry entry : accounting) { - + switch(entry.getEntryType()) { case CREATE: case UPDATE: case READ: Calendar calendar = entry.getDate(); - if(calendar.after(DateUtils.dateToCalendar(start)) && calendar.before(DateUtils.dateToCalendar(end))) { + if(calendar.after(DateUtils.dateToCalendar(start)) + && calendar.before(DateUtils.dateToCalendar(end))) { count++; } - + break; - + default: break; } - + } } catch(Exception e) { - logger.error("DataMethodDownloadHarvester: " + e.getLocalizedMessage()); - throw new InternalErrorException(e.getLocalizedMessage()); + throw new InternalErrorException(e); } } } - - + /** * Gets the VRE name to HL. * @@ -174,7 +177,7 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { */ private static String getVRENameToHL(String vre) { Validate.notNull(vre, "scope must be not null"); - + String newName; if(vre.startsWith(JCRRepository.PATH_SEPARATOR)) newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-").substring(1); @@ -182,5 +185,5 @@ public class DataMethodDownloadHarvester extends SoBigDataHarvester { newName = vre.replace(JCRRepository.PATH_SEPARATOR, "-"); return newName; } - + } diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java index 8d9a3dd..67d7a26 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java @@ -73,7 +73,7 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { //EXECUTING THE QUERY IN THE PERIOD String queryResult = executeQueryFor(solrParameters, start, end, "groups"); HarvestedDataKey insertDBKey = HarvestedDataKey.valueOf(mapSystemTypeToDBEntry.get(systemType)); - logger.info("Creating statistics for type {} using db key {}", systemType, insertDBKey); + logger.debug("Creating statistics for type {} using db key {}", systemType, insertDBKey); data.addAll(buildListOfHarvestedData(queryResult, insertDBKey)); } @@ -95,9 +95,7 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { JSONObject responseHeader = jsonObject.getJSONObject("responseHeader"); int status = responseHeader.getInt("status"); if(status != 0) { - String err = "Query Deliverable in error: status " + status; - logger.error(err); - throw new Exception(err); + throw new Exception("Query Deliverable in error: status " + status); } JSONObject response = jsonObject.getJSONObject("response"); @@ -139,7 +137,7 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { logger.debug("Document without groups, accounting it in the catalogue context"); catalogueContextCount++; } catch (Exception e) { - logger.error("Skipping parsing error", e); + logger.warn("Skipping parsing error", e); } } @@ -158,7 +156,7 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { data.add(new HarvestedData(harvestKey, mapCatalogueGroupToVRE.get(key), counter.get(key))); } - logger.info("For {} in the period [from {} to {}] returning accouting data :", harvestKey, DateUtils.format(start), DateUtils.format(end), data); + logger.debug("For {} in the period [from {} to {}] returning accouting data :", harvestKey, DateUtils.format(start), DateUtils.format(end), data); return data; diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index fa5e778..55c0252 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -43,7 +43,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { } - // @Test + @Test public void launch() { try { @@ -59,7 +59,7 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { inputs.put(AccountingDataHarvesterPlugin.MEASURE_TYPE_INPUT_PARAMETER, measureType.name()); inputs.put(AccountingDataHarvesterPlugin.RERUN_INPUT_PARAMETER, true); - inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); + inputs.put(AccountingDataHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, false); /* Calendar from = DateUtils.getStartCalendar(2018, Calendar.APRIL, 1);