diff --git a/pom.xml b/pom.xml index b596c37..9a75861 100644 --- a/pom.xml +++ b/pom.xml @@ -114,6 +114,11 @@ + + org.gcube.social-networking + social-service-client + [1.0.0, 2.0.0-SNAPSHOT) + org.json json diff --git a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java index ff1d075..e77c92d 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java @@ -18,6 +18,7 @@ import org.gcube.accounting.accounting.summary.access.model.internal.Dimension; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.authorization.utils.secret.GCubeSecret; import org.gcube.common.authorization.utils.secret.Secret; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; @@ -215,22 +216,17 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { Properties properties = getConfigParameters(); getProperties().set(properties); - ContextAuthorization contextAuthorization = new ContextAuthorization(properties); - - SortedSet contexts = contextAuthorization.getContexts(); - - String root = contexts.first(); - Utils.setContext(contextAuthorization.getSecretForContext(root)); - + String oldToken = properties.getProperty("OLD_ROOT_TOKEN"); + GCubeSecret gCubeSecret = new GCubeSecret(oldToken); + Utils.setContext(gCubeSecret); AccountingDao dao = AccountingDao.get(); - Set scopeDescriptorSet = dao.getContexts(); Map scopeDescriptorMap = new HashMap<>(); for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) { scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor); } scopeDescriptors.set(scopeDescriptorMap); - + Set dimensionSet = dao.getDimensions(); Map dimensionMap = new HashMap<>(); for (Dimension dimension : dimensionSet) { @@ -238,6 +234,15 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { } dimensions.set(dimensionMap); + + ContextAuthorization contextAuthorization = new ContextAuthorization(properties); + + SortedSet contexts = contextAuthorization.getContexts(); + + String root = contexts.first(); + Utils.setContext(contextAuthorization.getSecretForContext(root)); + + ArrayList accountingRecords = new ArrayList(); @@ -511,7 +516,8 @@ public class AccountingDashboardHarvesterPlugin extends Plugin { } } - Utils.setContext(rootSecret); + Utils.setContext(gCubeSecret); + // Utils.setContext(rootSecret); logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), accountingRecords); diff --git a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java index 09af8ec..4706653 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java @@ -37,23 +37,6 @@ public abstract class BasicHarvester { logger.debug("Creating {} for the period {} {} ", this.getClass().getSimpleName(), DateUtils.format(start), DateUtils.format(end)); } - public static String getCurrentContext(String token) throws Exception { - AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); - String context = authorizationEntry.getContext(); - logger.info("Context of token {} is {}", token, context); - return context; - } - - public static void setContext(String token) throws Exception { - SecurityTokenProvider.instance.set(token); - ScopeProvider.instance.set(getCurrentContext(token)); - } - - public static String getCurrentContext() throws Exception { - String token = SecurityTokenProvider.instance.get(); - return getCurrentContext(token); - } - public abstract List getAccountingRecords() throws Exception; public Dimension getDimension(HarvestedDataKey harvestedDataKey) { diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java index e1ab180..3c2357a 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java @@ -1,6 +1,5 @@ package org.gcube.dataharvest.harvester; -import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -10,8 +9,8 @@ import org.gcube.accounting.accounting.summary.access.model.update.AccountingRec import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; import org.gcube.dataharvest.datamodel.HarvestedDataKey; import org.gcube.dataharvest.utils.Utils; -import org.json.JSONArray; -import org.json.JSONObject; +import org.gcube.portal.databook.shared.Feed; +import org.gcube.social_networking.social_networking_client_library.PostClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,7 +18,7 @@ import org.slf4j.LoggerFactory; * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ -public class SocialInteractionsHarvester extends SocialNetworkingHarvester { +public class SocialInteractionsHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(SocialInteractionsHarvester.class); @@ -27,7 +26,7 @@ public class SocialInteractionsHarvester extends SocialNetworkingHarvester { private int replies; private int posts; - public static final String PATH = "/2/posts/get-posts-vre?gcube-token="; +// public static final String PATH = "/2/posts/get-posts-vre?gcube-token="; public SocialInteractionsHarvester(Date start, Date end) throws Exception { super(start, end); @@ -66,30 +65,20 @@ public class SocialInteractionsHarvester extends SocialNetworkingHarvester { } - - private void getJson() throws Exception { - JSONObject jsonObject = getJSONObject(PATH); - - Boolean success = (Boolean) jsonObject.get("success"); - if(success == false) { - throw new IOException("Erro while getting posts"); - } - - JSONArray res = jsonObject.getJSONArray("result"); - int len = res.length(); - + PostClient postClient = new PostClient(); + List vrePosts = postClient.getPostsVRE(); + likes = replies = posts = 0; - for(int i = 0; i < len; i++) { + for(Feed feed : vrePosts) { - JSONObject item = res.getJSONObject(i); - long time = item.getLong("time"); + long time = feed.getTime().getTime(); if(start.getTime() <= time && time <= end.getTime()) { posts++; - replies += item.getInt("comments_no"); - likes += item.getInt("likes_no"); + replies += Integer.valueOf(feed.getCommentsNo()); + likes += Integer.valueOf(feed.getLikesNo()); } } diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java deleted file mode 100644 index 1fe8962..0000000 --- a/src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.gcube.dataharvest.harvester; - -import java.util.Date; -import java.util.List; - -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.clients.exceptions.DiscoveryException; -import org.gcube.common.resources.gcore.GCoreEndpoint; -import org.gcube.dataharvest.utils.Utils; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -import org.gcube.resources.discovery.icclient.ICFactory; -import org.json.JSONObject; - -public abstract class SocialNetworkingHarvester extends BasicHarvester { - - public SocialNetworkingHarvester(Date start, Date end) throws Exception { - super(start, end); - } - - public static String CLASS_FORMAT = "$resource/Profile/ServiceClass/text() eq '%1s'"; - public static String NAME_FORMAT = "$resource/Profile/ServiceName/text() eq '%1s'"; - public static String STATUS_FORMAT = "$resource/Profile/DeploymentData/Status/text() eq 'ready'"; - public static String CONTAINS_FORMAT = "$entry/@EntryName eq '%1s'"; - - public static String SERVICE_CLASS = "Portal"; - public static String SERVICE_NAME = "SocialNetworking"; - public static String ENTRY_NAME = "jersey-servlet"; - - protected SimpleQuery getGCoreEndpointQuery() { - return ICFactory.queryFor(GCoreEndpoint.class) - .addCondition(String.format(CLASS_FORMAT, SERVICE_CLASS)) - .addCondition(String.format(NAME_FORMAT, SERVICE_NAME)) - .addCondition(String.format(STATUS_FORMAT)) - .addVariable("$entry", "$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint") - .addCondition(String.format(CONTAINS_FORMAT, ENTRY_NAME)) - .setResult("$entry/text()"); - } - - protected String getAddress() { - SimpleQuery gCoreEndpointQuery = getGCoreEndpointQuery(); - List addresses = ICFactory.client().submit(gCoreEndpointQuery); - if(addresses.size()==0) { - throw new DiscoveryException("No running Social Networking Service"); - } - return addresses.get(0); - } - - - protected JSONObject getJSONObject(String path) throws Exception { - String token = SecurityTokenProvider.instance.get(); - String baseAddress = getAddress(); - StringBuffer sb = new StringBuffer(baseAddress); - sb.append(path); - sb.append(token); - return new JSONObject(Utils.getJson(sb.toString())); - } - -} diff --git a/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java index ef9a062..b36270d 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java @@ -9,6 +9,7 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor; import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord; import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin; import org.gcube.dataharvest.datamodel.HarvestedDataKey; +import org.gcube.social_networking.social_networking_client_library.UserClient; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,7 +18,7 @@ import org.slf4j.LoggerFactory; * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ -public class VREUsersHarvester extends SocialNetworkingHarvester { +public class VREUsersHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(VREUsersHarvester.class); @@ -50,17 +51,8 @@ public class VREUsersHarvester extends SocialNetworkingHarvester { } private int get() throws Exception { - JSONObject jsonObject = getJSONObject(PATH); - - int userNumber = 0; - - Boolean success = (Boolean) jsonObject.get("success"); - if(success == false) { - throw new IOException("Erro while getting VRE Users"); - } - - userNumber = jsonObject.getJSONArray("result").length(); - return userNumber; + UserClient userClient = new UserClient(); + return userClient.getAllUsernamesContext().size(); } } diff --git a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java index ac12a4e..9ce050e 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java @@ -1,5 +1,11 @@ package org.gcube.dataharvest.harvester.sobigdata; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Date; @@ -210,10 +216,27 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester { query += "q=" + URLEncoder.encode(q, UTF_8_CHARASET) + "&wt=json&indent=true&rows=" + ROWS; query += flValue != null && !flValue.isEmpty() ? "&fl=" + URLEncoder.encode(flValue, UTF_8_CHARASET) : ""; logger.debug("\nPerforming query {}", query); - String jsonResult = Utils.getJson(query); + String jsonResult = requestJson(query); logger.trace("Response is {}", jsonResult); return jsonResult; } + + public String requestJson(String url) throws MalformedURLException, IOException { + URL address = new URL(url); + HttpURLConnection connection = (HttpURLConnection) address.openConnection(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + String json = ""; + String line = ""; + + while(line != null) { + line = reader.readLine(); + if(line != null) { + json += line.trim(); + } + } + return json; + } + } diff --git a/src/main/java/org/gcube/dataharvest/utils/Utils.java b/src/main/java/org/gcube/dataharvest/utils/Utils.java index f62e3a2..4527952 100644 --- a/src/main/java/org/gcube/dataharvest/utils/Utils.java +++ b/src/main/java/org/gcube/dataharvest/utils/Utils.java @@ -7,14 +7,10 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.client.exceptions.ObjectNotFound; -import org.gcube.common.authorization.library.AuthorizationEntry; -import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.utils.manager.SecretManager; import org.gcube.common.authorization.utils.manager.SecretManagerProvider; import org.gcube.common.authorization.utils.secret.Secret; -import org.gcube.common.scope.api.ScopeProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,31 +22,24 @@ public class Utils { private static Logger logger = LoggerFactory.getLogger(Utils.class); - public static String getJson(String url) throws MalformedURLException, IOException { - URL address = new URL(url); - HttpURLConnection connection = (HttpURLConnection) address.openConnection(); - BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); - String json = ""; - String line = ""; - - while(line != null) { - line = reader.readLine(); - if(line != null) { - json += line.trim(); - } - } - return json; - } +// public static String getJson(String url) throws MalformedURLException, IOException { +// URL address = new URL(url); +// HttpURLConnection connection = (HttpURLConnection) address.openConnection(); +// BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); +// String json = ""; +// String line = ""; +// +// while(line != null) { +// line = reader.readLine(); +// if(line != null) { +// json += line.trim(); +// } +// } +// return json; +// } public static String getCurrentContext() throws ObjectNotFound, Exception { - return getCurrentContext(SecurityTokenProvider.instance.get()); - } - - public static String getCurrentContext(String token) throws ObjectNotFound, Exception { - AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token); - String context = authorizationEntry.getContext(); - logger.info("Context of token {} is {}", token, context); - return context; + return SecretManagerProvider.instance.get().getContext(); } public static void setContext(Secret secret) throws Exception { @@ -61,9 +50,4 @@ public class Utils { secretManager.set(); } -// public static void setContext(String token) throws ObjectNotFound, Exception { -// SecurityTokenProvider.instance.set(token); -// ScopeProvider.instance.set(getCurrentContext(token)); -// } - } diff --git a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java index c63db26..9f920c7 100644 --- a/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java +++ b/src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java @@ -109,9 +109,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest { inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true); inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true); - inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, true); + inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false); - Calendar from = DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1); + Calendar from = DateUtils.getStartCalendar(2022, Calendar.AUGUST, 1); String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime()); logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate); inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);