gCubeSystem
/
accounting-dashboard-harvester-se-plugin
14
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Switched code to social-networking-client

This commit is contained in:
Luca Frosini 2022-09-05 14:49:45 +02:00
parent 6bd87cedc4
commit 459a71bc0d
9 changed files with 78 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -114,6 +114,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.gcube.social-networking</groupId>
<artifactId>social-service-client</artifactId>
<version>[1.0.0, 2.0.0-SNAPSHOT)</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>

26
src/main/java/org/gcube/dataharvest/AccountingDashboardHarvesterPlugin.java
View File

@ -18,6 +18,7 @@ import org.gcube.accounting.accounting.summary.access.model.internal.Dimension;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.com.fasterxml.jackson.annotation.JsonIgnore;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.utils.secret.GCubeSecret;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.scope.impl.ScopeBean;
import org.gcube.common.scope.impl.ScopeBean.Type;
@ -215,22 +216,17 @@ public class AccountingDashboardHarvesterPlugin extends Plugin {
Properties properties = getConfigParameters();
getProperties().set(properties);
ContextAuthorization contextAuthorization = new ContextAuthorization(properties);
SortedSet<String> contexts = contextAuthorization.getContexts();
String root = contexts.first();
Utils.setContext(contextAuthorization.getSecretForContext(root));
String oldToken = properties.getProperty("OLD_ROOT_TOKEN");
GCubeSecret gCubeSecret = new GCubeSecret(oldToken);
Utils.setContext(gCubeSecret);
AccountingDao dao = AccountingDao.get();
Set<ScopeDescriptor> scopeDescriptorSet = dao.getContexts();
Map<String, ScopeDescriptor> scopeDescriptorMap = new HashMap<>();
for (ScopeDescriptor scopeDescriptor : scopeDescriptorSet) {
scopeDescriptorMap.put(scopeDescriptor.getId(), scopeDescriptor);
}
scopeDescriptors.set(scopeDescriptorMap);
Set<Dimension> dimensionSet = dao.getDimensions();
Map<String, Dimension> dimensionMap = new HashMap<>();
for (Dimension dimension : dimensionSet) {
@ -238,6 +234,15 @@ public class AccountingDashboardHarvesterPlugin extends Plugin {
}
dimensions.set(dimensionMap);
ContextAuthorization contextAuthorization = new ContextAuthorization(properties);
SortedSet<String> contexts = contextAuthorization.getContexts();
String root = contexts.first();
Utils.setContext(contextAuthorization.getSecretForContext(root));
ArrayList<AccountingRecord> accountingRecords = new ArrayList<AccountingRecord>();
@ -511,7 +516,8 @@ public class AccountingDashboardHarvesterPlugin extends Plugin {
}
}
Utils.setContext(rootSecret);
Utils.setContext(gCubeSecret);
// Utils.setContext(rootSecret);
logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end),
accountingRecords);

17
src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java
View File

@ -37,23 +37,6 @@ public abstract class BasicHarvester {
logger.debug("Creating {} for the period {} {} ", this.getClass().getSimpleName(), DateUtils.format(start), DateUtils.format(end));
}
public static String getCurrentContext(String token) throws Exception {
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String context = authorizationEntry.getContext();
logger.info("Context of token {} is {}", token, context);
return context;
}
public static void setContext(String token) throws Exception {
SecurityTokenProvider.instance.set(token);
ScopeProvider.instance.set(getCurrentContext(token));
}
public static String getCurrentContext() throws Exception {
String token = SecurityTokenProvider.instance.get();
return getCurrentContext(token);
}
public abstract List<AccountingRecord> getAccountingRecords() throws Exception;
public Dimension getDimension(HarvestedDataKey harvestedDataKey) {

33
src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java
View File

@ -1,6 +1,5 @@
package org.gcube.dataharvest.harvester;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@ -10,8 +9,8 @@ import org.gcube.accounting.accounting.summary.access.model.update.AccountingRec
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.gcube.portal.databook.shared.Feed;
import org.gcube.social_networking.social_networking_client_library.PostClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -19,7 +18,7 @@ import org.slf4j.LoggerFactory;
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class SocialInteractionsHarvester extends SocialNetworkingHarvester {
public class SocialInteractionsHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(SocialInteractionsHarvester.class);
@ -27,7 +26,7 @@ public class SocialInteractionsHarvester extends SocialNetworkingHarvester {
private int replies;
private int posts;
public static final String PATH = "/2/posts/get-posts-vre?gcube-token=";
// public static final String PATH = "/2/posts/get-posts-vre?gcube-token=";
public SocialInteractionsHarvester(Date start, Date end) throws Exception {
super(start, end);
@ -66,30 +65,20 @@ public class SocialInteractionsHarvester extends SocialNetworkingHarvester {
}
private void getJson() throws Exception {
JSONObject jsonObject = getJSONObject(PATH);
Boolean success = (Boolean) jsonObject.get("success");
if(success == false) {
throw new IOException("Erro while getting posts");
}
JSONArray res = jsonObject.getJSONArray("result");
int len = res.length();
PostClient postClient = new PostClient();
List<Feed> vrePosts = postClient.getPostsVRE();
likes = replies = posts = 0;
for(int i = 0; i < len; i++) {
for(Feed feed : vrePosts) {
JSONObject item = res.getJSONObject(i);
long time = item.getLong("time");
long time = feed.getTime().getTime();
if(start.getTime() <= time && time <= end.getTime()) {
posts++;
replies += item.getInt("comments_no");
likes += item.getInt("likes_no");
replies += Integer.valueOf(feed.getCommentsNo());
likes += Integer.valueOf(feed.getLikesNo());
}
}

58
src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java
View File

@ -1,58 +0,0 @@
package org.gcube.dataharvest.harvester;
import java.util.Date;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.clients.exceptions.DiscoveryException;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
import org.json.JSONObject;
public abstract class SocialNetworkingHarvester extends BasicHarvester {
public SocialNetworkingHarvester(Date start, Date end) throws Exception {
super(start, end);
}
public static String CLASS_FORMAT = "$resource/Profile/ServiceClass/text() eq '%1s'";
public static String NAME_FORMAT = "$resource/Profile/ServiceName/text() eq '%1s'";
public static String STATUS_FORMAT = "$resource/Profile/DeploymentData/Status/text() eq 'ready'";
public static String CONTAINS_FORMAT = "$entry/@EntryName eq '%1s'";
public static String SERVICE_CLASS = "Portal";
public static String SERVICE_NAME = "SocialNetworking";
public static String ENTRY_NAME = "jersey-servlet";
protected SimpleQuery getGCoreEndpointQuery() {
return ICFactory.queryFor(GCoreEndpoint.class)
.addCondition(String.format(CLASS_FORMAT, SERVICE_CLASS))
.addCondition(String.format(NAME_FORMAT, SERVICE_NAME))
.addCondition(String.format(STATUS_FORMAT))
.addVariable("$entry", "$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint")
.addCondition(String.format(CONTAINS_FORMAT, ENTRY_NAME))
.setResult("$entry/text()");
}
protected String getAddress() {
SimpleQuery gCoreEndpointQuery = getGCoreEndpointQuery();
List<String> addresses = ICFactory.client().submit(gCoreEndpointQuery);
if(addresses.size()==0) {
throw new DiscoveryException("No running Social Networking Service");
}
return addresses.get(0);
}
protected JSONObject getJSONObject(String path) throws Exception {
String token = SecurityTokenProvider.instance.get();
String baseAddress = getAddress();
StringBuffer sb = new StringBuffer(baseAddress);
sb.append(path);
sb.append(token);
return new JSONObject(Utils.getJson(sb.toString()));
}
}

16
src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java
View File

@ -9,6 +9,7 @@ import org.gcube.accounting.accounting.summary.access.model.ScopeDescriptor;
import org.gcube.accounting.accounting.summary.access.model.update.AccountingRecord;
import org.gcube.dataharvest.AccountingDashboardHarvesterPlugin;
import org.gcube.dataharvest.datamodel.HarvestedDataKey;
import org.gcube.social_networking.social_networking_client_library.UserClient;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -17,7 +18,7 @@ import org.slf4j.LoggerFactory;
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class VREUsersHarvester extends SocialNetworkingHarvester {
public class VREUsersHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(VREUsersHarvester.class);
@ -50,17 +51,8 @@ public class VREUsersHarvester extends SocialNetworkingHarvester {
}
private int get() throws Exception {
JSONObject jsonObject = getJSONObject(PATH);
int userNumber = 0;
Boolean success = (Boolean) jsonObject.get("success");
if(success == false) {
throw new IOException("Erro while getting VRE Users");
}
userNumber = jsonObject.getJSONArray("result").length();
return userNumber;
UserClient userClient = new UserClient();
return userClient.getAllUsernamesContext().size();
}
}

25
src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -1,5 +1,11 @@
package org.gcube.dataharvest.harvester.sobigdata;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
@ -210,10 +216,27 @@ public class ResourceCatalogueHarvester extends SoBigDataHarvester {
query += "q=" + URLEncoder.encode(q, UTF_8_CHARASET) + "&wt=json&indent=true&rows=" + ROWS;
query += flValue != null && !flValue.isEmpty() ? "&fl=" + URLEncoder.encode(flValue, UTF_8_CHARASET) : "";
logger.debug("\nPerforming query {}", query);
String jsonResult = Utils.getJson(query);
String jsonResult = requestJson(query);
logger.trace("Response is {}", jsonResult);
return jsonResult;
}
public String requestJson(String url) throws MalformedURLException, IOException {
URL address = new URL(url);
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String json = "";
String line = "";
while(line != null) {
line = reader.readLine();
if(line != null) {
json += line.trim();
}
}
return json;
}
}

48
src/main/java/org/gcube/dataharvest/utils/Utils.java
View File

@ -7,14 +7,10 @@ import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.utils.manager.SecretManager;
import org.gcube.common.authorization.utils.manager.SecretManagerProvider;
import org.gcube.common.authorization.utils.secret.Secret;
import org.gcube.common.scope.api.ScopeProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -26,31 +22,24 @@ public class Utils {
private static Logger logger = LoggerFactory.getLogger(Utils.class);
public static String getJson(String url) throws MalformedURLException, IOException {
URL address = new URL(url);
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String json = "";
String line = "";
while(line != null) {
line = reader.readLine();
if(line != null) {
json += line.trim();
}
}
return json;
}
// public static String getJson(String url) throws MalformedURLException, IOException {
// URL address = new URL(url);
// HttpURLConnection connection = (HttpURLConnection) address.openConnection();
// BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
// String json = "";
// String line = "";
//
// while(line != null) {
// line = reader.readLine();
// if(line != null) {
// json += line.trim();
// }
// }
// return json;
// }
public static String getCurrentContext() throws ObjectNotFound, Exception {
return getCurrentContext(SecurityTokenProvider.instance.get());
}
public static String getCurrentContext(String token) throws ObjectNotFound, Exception {
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String context = authorizationEntry.getContext();
logger.info("Context of token {} is {}", token, context);
return context;
return SecretManagerProvider.instance.get().getContext();
}
public static void setContext(Secret secret) throws Exception {
@ -61,9 +50,4 @@ public class Utils {
secretManager.set();
}
// public static void setContext(String token) throws ObjectNotFound, Exception {
// SecurityTokenProvider.instance.set(token);
// ScopeProvider.instance.set(getCurrentContext(token));
// }
}

4
src/test/java/org/gcube/dataharvest/AccountingDataHarvesterPluginTest.java
View File

@ -109,9 +109,9 @@ public class AccountingDataHarvesterPluginTest extends ContextTest {
inputs.put(AccountingDashboardHarvesterPlugin.GET_VRE_USERS_INPUT_PARAMETER, true);
inputs.put(AccountingDashboardHarvesterPlugin.RERUN_INPUT_PARAMETER, true);
inputs.put(AccountingDashboardHarvesterPlugin.DRY_RUN_INPUT_PARAMETER, true);
inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, true);
inputs.put(AccountingDashboardHarvesterPlugin.PARTIAL_HARVESTING, false);
Calendar from = DateUtils.getStartCalendar(2021, Calendar.JANUARY, 1);
Calendar from = DateUtils.getStartCalendar(2022, Calendar.AUGUST, 1);
String fromDate = DateUtils.LAUNCH_DATE_FORMAT.format(from.getTime());
logger.trace("{} is {}", AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);
inputs.put(AccountingDashboardHarvesterPlugin.START_DATE_INPUT_PARAMETER, fromDate);