gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Getting Social Networking Service URL from IS 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167635 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-21 10:18:06 +00:00
parent 085ab9e9c2
commit 8edab20ef0
5 changed files with 108 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -12,7 +12,7 @@ import java.util.SortedSet;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
@ -40,8 +40,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
public static final String RERUN_INPUT_PARAMETER = "reRun";
public static final String DRY_RUN_INPUT_PARAMETER = "dryRun";
public static final String SOBIGDATA_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData";
public static final String SOBIGDATA_EU_CONTEXT = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
public static final String RESOURCE_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue";
public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe";
protected Date start;
@ -141,11 +141,11 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// Collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(start, end);
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end);
List<HarvestedData> harvested = socialHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Scoial Interactions for {}", context, e);
logger.error("Error harvesting Social Interactions for {}", context, e);
}
try {
@ -157,17 +157,17 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.error("Error harvesting Context Users for {}", context, e);
}
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
}
if(context.startsWith(SOBIGDATA_CONTEXT) || context.startsWith(SOBIGDATA_EU_CONTEXT)) {
if(context.startsWith(RESOURCE_CATALOGUE_CONTEXT)) {
try {
// Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception e) {
logger.error("Error harvesting Resource Catalogue Information for {}", context, e);
}
try {
// Collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);

41
src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java → src/main/java/org/gcube/dataharvest/harvester/SocialInteractionsHarvester.java
View File

@ -1,13 +1,11 @@
package org.gcube.dataharvest.harvester;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONArray;
@ -19,17 +17,18 @@ import org.slf4j.LoggerFactory;
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class SocialHarvester extends BasicHarvester {
public class SocialInteractionsHarvester extends SocialNetworkingHarvester {
private static Logger logger = LoggerFactory.getLogger(SocialHarvester.class);
private static Logger logger = LoggerFactory.getLogger(SocialInteractionsHarvester.class);
private int likes, replies, posts;
private long from = 0, to = 0;
private int likes;
private int replies;
private int posts;
public SocialHarvester(Date start, Date end) throws ParseException {
public static final String PATH = "/2/posts/get-posts-vre?gcube-token=";
public SocialInteractionsHarvester(Date start, Date end) throws ParseException {
super(start, end);
this.from = startDate.getTime();
this.to = endDate.getTime();
}
@Override
@ -63,34 +62,34 @@ public class SocialHarvester extends BasicHarvester {
}
private void getJson() throws MalformedURLException, IOException {
likes = replies = posts = 0;
// la seguente stringa deve essere letta dinamicamente
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token=";
String token = SecurityTokenProvider.instance.get();
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
private void getJson() throws Exception {
JSONObject jsonObject = getJSONObject(PATH);
Boolean success = (Boolean) jsonObject.get("success");
if(success == false) {
String message = "getJson() on token: " + token + " success=false";
logger.error(message);
throw new IOException(message);
throw new IOException("Erro while getting posts");
}
JSONArray res = jsonObject.getJSONArray("result");
int len = res.length();
likes = replies = posts = 0;
for(int i = 0; i < len; i++) {
JSONObject item = res.getJSONObject(i);
long time = item.getLong("time");
//System.out.println(from + " - " + time + " - " + to );
if((from <= time) && (time <= to)) {
if((startDate.getTime() <= time) && (time <= endDate.getTime())) {
posts++;
replies += item.getInt("comments_no");
likes += item.getInt("likes_no");
}
}
}
}

59
src/main/java/org/gcube/dataharvest/harvester/SocialNetworkingHarvester.java Normal file
View File

@ -0,0 +1,59 @@
package org.gcube.dataharvest.harvester;
import java.text.ParseException;
import java.util.Date;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.clients.exceptions.DiscoveryException;
import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.resources.discovery.icclient.ICFactory;
import org.json.JSONObject;
public abstract class SocialNetworkingHarvester extends BasicHarvester{
public SocialNetworkingHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
public static String CLASS_FORMAT = "$resource/Profile/ServiceClass/text() eq '%1s'";
public static String NAME_FORMAT = "$resource/Profile/ServiceName/text() eq '%1s'";
public static String STATUS_FORMAT = "$resource/Profile/DeploymentData/Status/text() eq 'ready'";
public static String CONTAINS_FORMAT = "$entry/@EntryName eq '%1s'";
public static String SERVICE_CLASS = "Portal";
public static String SERVICE_NAME = "SocialNetworking";
public static String ENTRY_NAME = "jersey-servlet";
protected SimpleQuery getGCoreEndpointQuery() {
return ICFactory.queryFor(GCoreEndpoint.class)
.addCondition(String.format(CLASS_FORMAT, SERVICE_CLASS))
.addCondition(String.format(NAME_FORMAT, SERVICE_NAME))
.addCondition(String.format(STATUS_FORMAT))
.addVariable("$entry", "$resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint")
.addCondition(String.format(CONTAINS_FORMAT, ENTRY_NAME))
.setResult("$entry/text()");
}
protected String getAddress() {
SimpleQuery gCoreEndpointQuery = getGCoreEndpointQuery();
List<String> addresses = ICFactory.client().submit(gCoreEndpointQuery);
if(addresses.size()==0) {
throw new DiscoveryException("No running Social Networking Service");
}
return addresses.get(0);
}
protected JSONObject getJSONObject(String path) throws Exception {
String token = SecurityTokenProvider.instance.get();
String baseAddress = getAddress();
StringBuffer sb = new StringBuffer(baseAddress);
sb.append(path);
sb.append(token);
return new JSONObject(Utils.getJson(sb.toString()));
}
}

21
src/main/java/org/gcube/dataharvest/harvester/VREUsersHarvester.java
View File

@ -1,13 +1,11 @@
package org.gcube.dataharvest.harvester;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONObject;
@ -18,10 +16,13 @@ import org.slf4j.LoggerFactory;
* @author Eric Perrone (ISTI - CNR)
* @author Luca Frosini (ISTI - CNR)
*/
public class VREUsersHarvester extends BasicHarvester {
public class VREUsersHarvester extends SocialNetworkingHarvester {
private static Logger logger = LoggerFactory.getLogger(VREUsersHarvester.class);
public static final String PATH = "/2/users/get-all-usernames?gcube-token=";
public VREUsersHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
@ -41,18 +42,16 @@ public class VREUsersHarvester extends BasicHarvester {
}
}
private int get() throws MalformedURLException, IOException {
private int get() throws Exception {
JSONObject jsonObject = getJSONObject(PATH);
int userNumber = 0;
// la seguente stringa deve essere letta dinamicamente
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/users/get-all-usernames?gcube-token=";
String token = SecurityTokenProvider.instance.get();
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
Boolean success = (Boolean) jsonObject.get("success");
if(success == false) {
String message = "get-all-usernames returned an error. token: " + token;
logger.error(message);
throw new IOException(message);
throw new IOException("Erro while getting VRE Users");
}
userNumber = jsonObject.getJSONArray("result").length();
return userNumber;
}

6
src/test/java/org/gcube/dataharvest/Harvester.java
View File

@ -11,7 +11,7 @@ import org.gcube.dataharvest.dao.DatabaseConnectionData;
import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.SocialInteractionsHarvester;
import org.gcube.dataharvest.harvester.VREUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
@ -37,7 +37,7 @@ public class Harvester {
Harvester harvester = new Harvester();
try {
harvester.processParameterArray(args);
SocialHarvester socialHarvester = new SocialHarvester(harvester.getDateFrom(), harvester.getDateTo());
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(harvester.getDateFrom(), harvester.getDateTo());
harvester.runOne(socialHarvester);
// harvester.run();
//ArrayList<Integer> list = harvester.getSubTree(17);
@ -116,7 +116,7 @@ public class Harvester {
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo);
SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(dateFrom, dateTo);
List<HarvestedData> res = socialHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch(Exception x) {