diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index 21cc835..37089b2 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -2,21 +2,24 @@ package org.gcube.dataharvest; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.SortedSet; import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.Harvest; -import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester; -import org.gcube.dataharvest.harvester.MethodInvocationHarvester; -import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.SocialHarvester; import org.gcube.dataharvest.harvester.VreUsersHarvester; +import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; +import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.MeasureType; +import org.gcube.dataharvest.utils.Utils; import org.gcube.vremanagement.executor.plugin.Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +38,10 @@ public class AccountingDataHarvesterPlugin extends Plugin properties = new InheritableThreadLocal() { @Override @@ -123,28 +130,31 @@ public class AccountingDataHarvesterPlugin extends Plugin contexts = contextAuthorization.getContexts(); - for() { + ArrayList data = new ArrayList(); + + for(String context : contexts) { + + // Setting the token for the context + Utils.setContext(contextAuthorization.getTokenForContext(context)); try { // collecting info on VRE users VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end); - List users = vreUsersHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, users, reRun); - } + List harvested = vreUsersHarvester.getData(); + data.addAll(harvested); } catch(Exception x) { logger.error(x.getLocalizedMessage()); } + try { // collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods) ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); - List res = resourceCatalogueHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); - } + List harvested = resourceCatalogueHarvester.getData(); + data.addAll(harvested); } catch(Exception x) { logger.error(x.getLocalizedMessage()); } @@ -152,10 +162,8 @@ public class AccountingDataHarvesterPlugin extends Plugin res = dataMethodDownloadHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); - } + List harvested = dataMethodDownloadHarvester.getData(); + data.addAll(harvested); } catch(Exception x) { logger.error(x.getLocalizedMessage()); } @@ -163,10 +171,8 @@ public class AccountingDataHarvesterPlugin extends Plugin res = socialHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); - } + List harvested = socialHarvester.getData(); + data.addAll(harvested); } catch(Exception x) { logger.error(x.getLocalizedMessage()); } @@ -174,14 +180,17 @@ public class AccountingDataHarvesterPlugin extends Plugin res = methodInvocationHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); - } + List harvested = methodInvocationHarvester.getData(); + data.addAll(harvested); } catch(Exception x) { logger.error(x.getLocalizedMessage()); } } + + if(!dryRun) { + dbaseManager.insertMonthlyData(start, end, data, reRun); + } + } /** {@inheritDoc} */ diff --git a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java index 3e09bcc..77db1df 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java @@ -8,20 +8,18 @@ import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.Harvest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BasicHarvester { +public abstract class BasicHarvester { private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class); public Date startDate; public Date endDate; - public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester"; - public final String CATEGORY_NAME = "Accounting"; + public BasicHarvester(Date start, Date end) throws ParseException { startDate = start; @@ -46,13 +44,6 @@ public class BasicHarvester { return getCurrentContext(token); } - public List getData() throws Exception { - return null; - } + public abstract List getData() throws Exception; - public String[] getActiveVREs(boolean testMode) { - DatabaseManager dbaseManager = new DatabaseManager(); - return dbaseManager.getActiveVres(); - } - } diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java index 00abb8a..82ebb53 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java @@ -6,8 +6,8 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Properties; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.utils.Utils; import org.json.JSONArray; @@ -30,39 +30,41 @@ public class SocialHarvester extends BasicHarvester { @Override public List getData() throws Exception { - ArrayList data = new ArrayList(); - String[] vres = getActiveVREs(true); - for(String vre : vres) { - try { - logger.debug("Working on VRE: " + vre); - - String key = vre; // vre.substring(vre.lastIndexOf("/") + 1); - String token = props.getProperty(key); - //System.out.println(key + ":" + token); - getJson(token); - //getFinalResult(); - Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes); - data.add(harvest); - logger.debug(harvest.toString()); - - harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts); - data.add(harvest); - logger.debug(harvest.toString()); - - harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies); - data.add(harvest); - logger.debug(harvest.toString()); - } catch(Exception x) { - logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage()); - } + + String context = Utils.getCurrentContext(); + + try { + + ArrayList data = new ArrayList(); + + getJson(); + + Harvest likesH = new Harvest(Harvest.SOCIAL_LIKES, context, likes); + logger.debug("{}", likesH); + data.add(likesH); + + Harvest postsH = new Harvest(Harvest.SOCIAL_POSTS, context, posts); + logger.debug("{}", postsH); + data.add(postsH); + + Harvest socialReplies = new Harvest(Harvest.SOCIAL_REPLIES, context, replies); + logger.debug("{}", socialReplies); + data.add(socialReplies); + + return data; + } catch(Exception e) { + logger.error("Error Harvesting Social Interactions for context {}", context, e); + throw e; } - return data; + } - private void getJson(String token) throws MalformedURLException, IOException { + private void getJson() throws MalformedURLException, IOException { + likes = replies = posts = 0; // la seguente stringa deve essere letta dinamicamente String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token="; + String token = SecurityTokenProvider.instance.get(); JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); Boolean success = (Boolean) jsonObject.get("success"); @@ -87,6 +89,4 @@ public class SocialHarvester extends BasicHarvester { } } - - } diff --git a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java index 25f32d6..c1d9dae 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java @@ -6,8 +6,8 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Properties; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.utils.Utils; import org.json.JSONObject; @@ -25,28 +25,25 @@ public class VreUsersHarvester extends BasicHarvester { @Override public List getData() throws Exception { - ArrayList data = new ArrayList(); - Properties props = readServiceEndpoint(); - String[] vres = getActiveVREs(true); - for (String vre : vres) { - String key = vre.substring(vre.lastIndexOf("/") + 1); - String token = props.getProperty(key); - try { - int measure = get(token); - Harvest harvest = new Harvest(Harvest.USERS, vre, measure); - data.add(harvest); - logger.debug(harvest.toString()); - } catch (Exception x) { - logger.error("VreUsersHarvester::getData. " + vre + ". " + x.getLocalizedMessage()); - } + String context = Utils.getCurrentContext(); + try { + int measure = get(); + Harvest harvest = new Harvest(Harvest.USERS, context, measure); + logger.debug(harvest.toString()); + ArrayList data = new ArrayList(); + data.add(harvest); + return data; + } catch (Exception e) { + logger.error("Error Haversting Context Users for context {}", context, e); + throw e; } - return data; } - private int get(String token) throws MalformedURLException, IOException { + private int get() throws MalformedURLException, IOException { int userNumber = 0; // la seguente stringa deve essere letta dinamicamente String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/users/get-all-usernames?gcube-token="; + String token = SecurityTokenProvider.instance.get(); JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); Boolean success = (Boolean) jsonObject.get("success"); if (success == false) { diff --git a/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java similarity index 66% rename from src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java rename to src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java index ca45e56..82f8cd2 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java @@ -1,4 +1,4 @@ -package org.gcube.dataharvest.harvester; +package org.gcube.dataharvest.harvester.sobigdata; import java.text.ParseException; import java.util.ArrayList; @@ -16,8 +16,8 @@ import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; import org.gcube.common.homelibrary.jcr.repository.JCRRepository; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem; -import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.harvester.BasicHarvester; import org.gcube.dataharvest.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,9 +25,7 @@ import org.slf4j.LoggerFactory; public class DataMethodDownloadHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class); - - private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu"; - + private int count = 0; public DataMethodDownloadHarvester(Date start, Date end) throws ParseException { @@ -36,40 +34,39 @@ public class DataMethodDownloadHarvester extends BasicHarvester { @Override public List getData() throws Exception { - ArrayList data = new ArrayList(); + String context = Utils.getCurrentContext(); - ScopeProvider.instance.set(SCOPE); - - String[] vres = getActiveVREs(true); - for (String vre : vres) { - try { - count = 0; - - HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager(); + try { + ArrayList data = new ArrayList(); + + count = 0; + + HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager(); - String user = getVREName(vre) + "-Manager"; - - Home home = manager.getHome(user); - - JCRWorkspace ws = (JCRWorkspace) home.getWorkspace(); - - JCRWorkspaceItem item = (JCRWorkspaceItem) ws - .getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre)); + String user = getVREName(context) + "-Manager"; + + Home home = manager.getHome(user); + + JCRWorkspace ws = (JCRWorkspace) home.getWorkspace(); + + JCRWorkspaceItem item = (JCRWorkspaceItem) ws + .getItemByPath("/Workspace/MySpecialFolders/" + getVREName(context)); - logger.debug("Analyzing " + vre + " from " + startDate.toString() + " to " + endDate.toString()); + logger.debug("Analyzing " + context + " from " + startDate.toString() + " to " + endDate.toString()); - logger.error("Before getStats()"); - getStats(item, startDate, endDate); - logger.error("After getStats()"); - Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, vre, count); - data.add(harvest); - logger.debug(harvest.toString()); - } catch (Exception e) { - logger.error("DataMethodDownloadHarvester: Error processing VRE: " + vre + "\n" + - e.getLocalizedMessage()); - } + logger.error("Before getStats()"); + getStats(item, startDate, endDate); + logger.error("After getStats()"); + Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, context, count); + data.add(harvest); + logger.debug(harvest.toString()); + return data; + + } catch (Exception e) { + logger.error("Error Harvesting Data Methods Download for context {}", context, e); + throw e; } - return data; + } diff --git a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java similarity index 96% rename from src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java rename to src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java index 5a9784e..dbcff8d 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java @@ -1,4 +1,4 @@ -package org.gcube.dataharvest.harvester; +package org.gcube.dataharvest.harvester.sobigdata; import java.text.ParseException; import java.util.ArrayList; @@ -18,6 +18,7 @@ import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFact import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.harvester.BasicHarvester; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java similarity index 97% rename from src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java rename to src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java index a404c76..2fa2b55 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java @@ -1,4 +1,4 @@ -package org.gcube.dataharvest.harvester; +package org.gcube.dataharvest.harvester.sobigdata; import java.text.ParseException; import java.util.ArrayList; @@ -7,6 +7,7 @@ import java.util.Iterator; import java.util.List; import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.harvester.BasicHarvester; import org.gcube.dataharvest.utils.Utils; import org.gcube.portlets.user.urlshortener.UrlEncoderUtil; import org.json.JSONArray; diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java index 9d1f70b..8d020e9 100644 --- a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java +++ b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java @@ -9,7 +9,8 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; -import java.util.TreeMap; +import java.util.SortedSet; +import java.util.TreeSet; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.UserInfo; @@ -49,7 +50,7 @@ public class ContextAuthorization { * Contains Properties used to generate tokens */ public ContextAuthorization() throws Exception { - this.contextToToken = new TreeMap<>(); + this.contextToToken = new HashMap<>(); this.tokenToContext = new HashMap<>(); retrieveContextsAndTokens(); } @@ -116,6 +117,9 @@ public class ContextAuthorization { return tokenToContext.get(token); } + public SortedSet getContexts(){ + return new TreeSet(contextToToken.keySet()); + } diff --git a/src/main/java/org/gcube/dataharvest/utils/Utils.java b/src/main/java/org/gcube/dataharvest/utils/Utils.java index 15be59b..ce8ca38 100644 --- a/src/main/java/org/gcube/dataharvest/utils/Utils.java +++ b/src/main/java/org/gcube/dataharvest/utils/Utils.java @@ -11,6 +11,11 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import org.gcube.common.authorization.client.Constants; +import org.gcube.common.authorization.client.exceptions.ObjectNotFound; +import org.gcube.common.authorization.library.AuthorizationEntry; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.scope.api.ScopeProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,5 +49,21 @@ public class Utils { } return json; } - + + public static String getCurrentContext() throws ObjectNotFound, Exception{ + return getCurrentContext(SecurityTokenProvider.instance.get()); + } + + public static String getCurrentContext(String token) throws ObjectNotFound, Exception{ + AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); + String context = authorizationEntry.getContext(); + logger.info("Context of token {} is {}", token, context); + return context; + } + + + public static void setContext(String token) throws ObjectNotFound, Exception{ + SecurityTokenProvider.instance.set(token); + ScopeProvider.instance.set(getCurrentContext(token)); + } } diff --git a/src/test/java/org/gcube/dataharvest/Harvester.java b/src/test/java/org/gcube/dataharvest/Harvester.java index c386bd4..b58cacb 100644 --- a/src/test/java/org/gcube/dataharvest/Harvester.java +++ b/src/test/java/org/gcube/dataharvest/Harvester.java @@ -11,11 +11,11 @@ import org.gcube.dataharvest.dao.DatabaseConnectionData; import org.gcube.dataharvest.dao.DatabaseParameterRetriever; import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.harvester.BasicHarvester; -import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester; -import org.gcube.dataharvest.harvester.MethodInvocationHarvester; -import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.SocialHarvester; import org.gcube.dataharvest.harvester.VreUsersHarvester; +import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; +import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester; +import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/org/gcube/dataharvest/utils/ScopedTest.java b/src/test/java/org/gcube/dataharvest/utils/ScopedTest.java index e262e7b..8ae8766 100644 --- a/src/test/java/org/gcube/dataharvest/utils/ScopedTest.java +++ b/src/test/java/org/gcube/dataharvest/utils/ScopedTest.java @@ -76,22 +76,9 @@ public class ScopedTest { DEFAULT_TEST_SCOPE = GCUBE; } - public static String getCurrentScope(String token) throws ObjectNotFound, Exception{ - AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); - String context = authorizationEntry.getContext(); - logger.info("Context of token {} is {}", token, context); - return context; - } - - - public static void setContext(String token) throws ObjectNotFound, Exception{ - SecurityTokenProvider.instance.set(token); - ScopeProvider.instance.set(getCurrentScope(token)); - } - @BeforeClass public static void beforeClass() throws Exception{ - setContext(DEFAULT_TEST_SCOPE); + Utils.setContext(DEFAULT_TEST_SCOPE); } @AfterClass