From 8edab20ef0f2c02b8ad115c90ccbe6d8778e5ed7 Mon Sep 17 00:00:00 2001 From: "luca.frosini" Date: Mon, 21 May 2018 10:18:06 +0000 Subject: [PATCH] Getting Social Networking Service URL from IS Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167635 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../AccountingDataHarvesterPlugin.java | 32 +++++----- ....java => SocialInteractionsHarvester.java} | 41 +++++++------ .../harvester/SocialNetworkingHarvester.java | 59 +++++++++++++++++++ .../harvester/VREUsersHarvester.java | 21 ++++--- .../java/org/gcube/dataharvest/Harvester.java | 6 +- 5 files changed, 108 insertions(+), 51 deletions(-) rename src/main/java/org/gcube/dataharvest/harvester/{SocialHarvester.java => SocialInteractionsHarvester.java} (62%) create mode 100644 src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index 6226378..35aacb7 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -12,7 +12,7 @@ import java.util.SortedSet; import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; -import org.gcube.dataharvest.harvester.SocialHarvester; +import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; @@ -40,8 +40,8 @@ public class AccountingDataHarvesterPlugin extends Plugin harvested = socialHarvester.getData(); data.addAll(harvested); } catch(Exception e) { - logger.error("Error harvesting Scoial Interactions for {}", context, e); + logger.error("Error harvesting Social Interactions for {}", context, e); } try { @@ -157,17 +157,17 @@ public class AccountingDataHarvesterPlugin extends Plugin harvested = resourceCatalogueHarvester.getData(); - data.addAll(harvested); - } catch(Exception e) { - logger.error("Error harvesting Resource Catalogue Information for {}", context, e); - } - - - if(context.startsWith(SOBIGDATA_CONTEXT) || context.startsWith(SOBIGDATA_EU_CONTEXT)) { + if(context.startsWith(RESOURCE_CATALOGUE_CONTEXT)) { + + try { + // Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods) + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); + List harvested = resourceCatalogueHarvester.getData(); + data.addAll(harvested); + } catch(Exception e) { + logger.error("Error harvesting Resource Catalogue Information for {}", context, e); + } + try { // Collecting info on Data/Method download DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end); diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java similarity index 62% rename from src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java rename to src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java index 3bb6a30..efb9db9 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java @@ -1,13 +1,11 @@ package org.gcube.dataharvest.harvester; import java.io.IOException; -import java.net.MalformedURLException; import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.List; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.utils.Utils; import org.json.JSONArray; @@ -19,17 +17,18 @@ import org.slf4j.LoggerFactory; * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ -public class SocialHarvester extends BasicHarvester { +public class SocialInteractionsHarvester extends SocialNetworkingHarvester { - private static Logger logger = LoggerFactory.getLogger(SocialHarvester.class); + private static Logger logger = LoggerFactory.getLogger(SocialInteractionsHarvester.class); - private int likes, replies, posts; - private long from = 0, to = 0; + private int likes; + private int replies; + private int posts; - public SocialHarvester(Date start, Date end) throws ParseException { + public static final String PATH = "/2/posts/get-posts-vre?gcube-token="; + + public SocialInteractionsHarvester(Date start, Date end) throws ParseException { super(start, end); - this.from = startDate.getTime(); - this.to = endDate.getTime(); } @Override @@ -63,34 +62,34 @@ public class SocialHarvester extends BasicHarvester { } - private void getJson() throws MalformedURLException, IOException { - - likes = replies = posts = 0; - // la seguente stringa deve essere letta dinamicamente - String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token="; - String token = SecurityTokenProvider.instance.get(); - JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); + + + private void getJson() throws Exception { + JSONObject jsonObject = getJSONObject(PATH); Boolean success = (Boolean) jsonObject.get("success"); if(success == false) { - String message = "getJson() on token: " + token + " success=false"; - logger.error(message); - throw new IOException(message); + throw new IOException("Erro while getting posts"); } JSONArray res = jsonObject.getJSONArray("result"); int len = res.length(); + likes = replies = posts = 0; + for(int i = 0; i < len; i++) { + JSONObject item = res.getJSONObject(i); long time = item.getLong("time"); - //System.out.println(from + " - " + time + " - " + to ); - if((from <= time) && (time <= to)) { + + if((startDate.getTime() <= time) && (time <= endDate.getTime())) { posts++; replies += item.getInt("comments_no"); likes += item.getInt("likes_no"); } + } + } } diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java new file mode 100644 index 0000000..3aee2e2 --- /dev/null +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java @@ -0,0 +1,59 @@ +package org.gcube.dataharvest.harvester; + +import java.text.ParseException; +import java.util.Date; +import java.util.List; + +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.clients.exceptions.DiscoveryException; +import org.gcube.common.resources.gcore.GCoreEndpoint; +import org.gcube.dataharvest.utils.Utils; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.gcube.resources.discovery.icclient.ICFactory; +import org.json.JSONObject; + +public abstract class SocialNetworkingHarvester extends BasicHarvester{ + + public SocialNetworkingHarvester(Date start, Date end) throws ParseException { + super(start, end); + } + + public static String CLASS_FORMAT = "$resource/Profile/ServiceClass/text() eq '%1s'"; + public static String NAME_FORMAT = "$resource/Profile/ServiceName/text() eq '%1s'"; + public static String STATUS_FORMAT = "$resource/Profile/DeploymentData/Status/text() eq 'ready'"; + public static String CONTAINS_FORMAT = "$entry/@EntryName eq '%1s'"; + + public static String SERVICE_CLASS = "Portal"; + public static String SERVICE_NAME = "SocialNetworking"; + public static String ENTRY_NAME = "jersey-servlet"; + + protected SimpleQuery getGCoreEndpointQuery() { + return ICFactory.queryFor(GCoreEndpoint.class) + .addCondition(String.format(CLASS_FORMAT, SERVICE_CLASS)) + .addCondition(String.format(NAME_FORMAT, SERVICE_NAME)) + .addCondition(String.format(STATUS_FORMAT)) + .addVariable("$entry", "$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint") + .addCondition(String.format(CONTAINS_FORMAT, ENTRY_NAME)) + .setResult("$entry/text()"); + } + + protected String getAddress() { + SimpleQuery gCoreEndpointQuery = getGCoreEndpointQuery(); + List addresses = ICFactory.client().submit(gCoreEndpointQuery); + if(addresses.size()==0) { + throw new DiscoveryException("No running Social Networking Service"); + } + return addresses.get(0); + } + + + protected JSONObject getJSONObject(String path) throws Exception { + String token = SecurityTokenProvider.instance.get(); + String baseAddress = getAddress(); + StringBuffer sb = new StringBuffer(baseAddress); + sb.append(path); + sb.append(token); + return new JSONObject(Utils.getJson(sb.toString())); + } + +} diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java index ef40e56..25f0886 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java @@ -1,13 +1,11 @@ package org.gcube.dataharvest.harvester; import java.io.IOException; -import java.net.MalformedURLException; import java.text.ParseException; import java.util.ArrayList; import java.util.Date; import java.util.List; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.utils.Utils; import org.json.JSONObject; @@ -18,10 +16,13 @@ import org.slf4j.LoggerFactory; * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ -public class VREUsersHarvester extends BasicHarvester { +public class VREUsersHarvester extends SocialNetworkingHarvester { private static Logger logger = LoggerFactory.getLogger(VREUsersHarvester.class); + + public static final String PATH = "/2/users/get-all-usernames?gcube-token="; + public VREUsersHarvester(Date start, Date end) throws ParseException { super(start, end); } @@ -41,18 +42,16 @@ public class VREUsersHarvester extends BasicHarvester { } } - private int get() throws MalformedURLException, IOException { + private int get() throws Exception { + JSONObject jsonObject = getJSONObject(PATH); + int userNumber = 0; - // la seguente stringa deve essere letta dinamicamente - String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/users/get-all-usernames?gcube-token="; - String token = SecurityTokenProvider.instance.get(); - JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); + Boolean success = (Boolean) jsonObject.get("success"); if(success == false) { - String message = "get-all-usernames returned an error. token: " + token; - logger.error(message); - throw new IOException(message); + throw new IOException("Erro while getting VRE Users"); } + userNumber = jsonObject.getJSONArray("result").length(); return userNumber; } diff --git a/src/test/java/org/gcube/dataharvest/Harvester.java b/src/test/java/org/gcube/dataharvest/Harvester.java index 1fa87cb..3a910f7 100644 --- a/src/test/java/org/gcube/dataharvest/Harvester.java +++ b/src/test/java/org/gcube/dataharvest/Harvester.java @@ -11,7 +11,7 @@ import org.gcube.dataharvest.dao.DatabaseConnectionData; import org.gcube.dataharvest.dao.DatabaseParameterRetriever; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.BasicHarvester; -import org.gcube.dataharvest.harvester.SocialHarvester; +import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; @@ -37,7 +37,7 @@ public class Harvester { Harvester harvester = new Harvester(); try { harvester.processParameterArray(args); - SocialHarvester socialHarvester = new SocialHarvester(harvester.getDateFrom(), harvester.getDateTo()); + SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(harvester.getDateFrom(), harvester.getDateTo()); harvester.runOne(socialHarvester); // harvester.run(); //ArrayList list = harvester.getSubTree(17); @@ -116,7 +116,7 @@ public class Harvester { try { // collecting info on social (posts, replies and likes) - SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo); + SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(dateFrom, dateTo); List res = socialHarvester.getData(); insertMonthlyData((Date) dateFrom, (Date) dateTo, res); } catch(Exception x) {