gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167585 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-18 10:00:10 +00:00
parent 8d64022afa
commit 0f3b397fbd
11 changed files with 147 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -2,21 +2,24 @@ package org.gcube.dataharvest;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.SortedSet;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VreUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.gcube.dataharvest.utils.ContextAuthorization;
import org.gcube.dataharvest.utils.DateUtils;
import org.gcube.dataharvest.utils.MeasureType;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.vremanagement.executor.plugin.Plugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -35,6 +38,10 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
protected Date start;
protected Date end;
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
super(pluginDeclaration);
}
private static final InheritableThreadLocal<Properties> properties = new InheritableThreadLocal<Properties>() {
@Override
@ -123,28 +130,31 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
DatabaseManager dbaseManager = new DatabaseManager();
SortedSet<String> contexts = contextAuthorization.getContexts();
for() {
ArrayList<Harvest> data = new ArrayList<Harvest>();
for(String context : contexts) {
// Setting the token for the context
Utils.setContext(contextAuthorization.getTokenForContext(context));
try {
// collecting info on VRE users
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
List<Harvest> users = vreUsersHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, users, reRun);
}
List<Harvest> harvested = vreUsersHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<Harvest> res = resourceCatalogueHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
List<Harvest> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
@ -152,10 +162,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
List<Harvest> res = dataMethodDownloadHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
List<Harvest> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
@ -163,10 +171,8 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(start, end);
List<Harvest> res = socialHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
List<Harvest> harvested = socialHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
@ -174,14 +180,17 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<Harvest> res = methodInvocationHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
List<Harvest> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
}
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, data, reRun);
}
}
/** {@inheritDoc} */

15
src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java
View File

@ -8,20 +8,18 @@ import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.Harvest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BasicHarvester {
public abstract class BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class);
public Date startDate;
public Date endDate;
public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester";
public final String CATEGORY_NAME = "Accounting";
public BasicHarvester(Date start, Date end) throws ParseException {
startDate = start;
@ -46,13 +44,6 @@ public class BasicHarvester {
return getCurrentContext(token);
}
public List<Harvest> getData() throws Exception {
return null;
}
public abstract List<Harvest> getData() throws Exception;
public String[] getActiveVREs(boolean testMode) {
DatabaseManager dbaseManager = new DatabaseManager();
return dbaseManager.getActiveVres();
}
}

60
src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java
View File

@ -6,8 +6,8 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONArray;
@ -30,39 +30,41 @@ public class SocialHarvester extends BasicHarvester {
@Override
public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
String[] vres = getActiveVREs(true);
for(String vre : vres) {
try {
logger.debug("Working on VRE: " + vre);
String key = vre; // vre.substring(vre.lastIndexOf("/") + 1);
String token = props.getProperty(key);
//System.out.println(key + ":" + token);
getJson(token);
//getFinalResult();
Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes);
data.add(harvest);
logger.debug(harvest.toString());
harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts);
data.add(harvest);
logger.debug(harvest.toString());
harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies);
data.add(harvest);
logger.debug(harvest.toString());
} catch(Exception x) {
logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage());
}
String context = Utils.getCurrentContext();
try {
ArrayList<Harvest> data = new ArrayList<Harvest>();
getJson();
Harvest likesH = new Harvest(Harvest.SOCIAL_LIKES, context, likes);
logger.debug("{}", likesH);
data.add(likesH);
Harvest postsH = new Harvest(Harvest.SOCIAL_POSTS, context, posts);
logger.debug("{}", postsH);
data.add(postsH);
Harvest socialReplies = new Harvest(Harvest.SOCIAL_REPLIES, context, replies);
logger.debug("{}", socialReplies);
data.add(socialReplies);
return data;
} catch(Exception e) {
logger.error("Error Harvesting Social Interactions for context {}", context, e);
throw e;
}
return data;
}
private void getJson(String token) throws MalformedURLException, IOException {
private void getJson() throws MalformedURLException, IOException {
likes = replies = posts = 0;
// la seguente stringa deve essere letta dinamicamente
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token=";
String token = SecurityTokenProvider.instance.get();
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
Boolean success = (Boolean) jsonObject.get("success");
@ -87,6 +89,4 @@ public class SocialHarvester extends BasicHarvester {
}
}
}

31
src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java
View File

@ -6,8 +6,8 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONObject;
@ -25,28 +25,25 @@ public class VreUsersHarvester extends BasicHarvester {
@Override
public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
Properties props = readServiceEndpoint();
String[] vres = getActiveVREs(true);
for (String vre : vres) {
String key = vre.substring(vre.lastIndexOf("/") + 1);
String token = props.getProperty(key);
try {
int measure = get(token);
Harvest harvest = new Harvest(Harvest.USERS, vre, measure);
data.add(harvest);
logger.debug(harvest.toString());
} catch (Exception x) {
logger.error("VreUsersHarvester::getData. " + vre + ". " + x.getLocalizedMessage());
}
String context = Utils.getCurrentContext();
try {
int measure = get();
Harvest harvest = new Harvest(Harvest.USERS, context, measure);
logger.debug(harvest.toString());
ArrayList<Harvest> data = new ArrayList<Harvest>();
data.add(harvest);
return data;
} catch (Exception e) {
logger.error("Error Haversting Context Users for context {}", context, e);
throw e;
}
return data;
}
private int get(String token) throws MalformedURLException, IOException {
private int get() throws MalformedURLException, IOException {
int userNumber = 0;
// la seguente stringa deve essere letta dinamicamente
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/users/get-all-usernames?gcube-token=";
String token = SecurityTokenProvider.instance.get();
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
Boolean success = (Boolean) jsonObject.get("success");
if (success == false) {

65
src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java → src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest.harvester;
package org.gcube.dataharvest.harvester.sobigdata;
import java.text.ParseException;
import java.util.ArrayList;
@ -16,8 +16,8 @@ import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -25,9 +25,7 @@ import org.slf4j.LoggerFactory;
public class DataMethodDownloadHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class);
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
private int count = 0;
public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
@ -36,40 +34,39 @@ public class DataMethodDownloadHarvester extends BasicHarvester {
@Override
public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
String context = Utils.getCurrentContext();
ScopeProvider.instance.set(SCOPE);
String[] vres = getActiveVREs(true);
for (String vre : vres) {
try {
count = 0;
HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager();
try {
ArrayList<Harvest> data = new ArrayList<Harvest>();
count = 0;
HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager();
String user = getVREName(vre) + "-Manager";
Home home = manager.getHome(user);
JCRWorkspace ws = (JCRWorkspace) home.getWorkspace();
JCRWorkspaceItem item = (JCRWorkspaceItem) ws
.getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre));
String user = getVREName(context) + "-Manager";
Home home = manager.getHome(user);
JCRWorkspace ws = (JCRWorkspace) home.getWorkspace();
JCRWorkspaceItem item = (JCRWorkspaceItem) ws
.getItemByPath("/Workspace/MySpecialFolders/" + getVREName(context));
logger.debug("Analyzing " + vre + " from " + startDate.toString() + " to " + endDate.toString());
logger.debug("Analyzing " + context + " from " + startDate.toString() + " to " + endDate.toString());
logger.error("Before getStats()");
getStats(item, startDate, endDate);
logger.error("After getStats()");
Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, vre, count);
data.add(harvest);
logger.debug(harvest.toString());
} catch (Exception e) {
logger.error("DataMethodDownloadHarvester: Error processing VRE: " + vre + "\n" +
e.getLocalizedMessage());
}
logger.error("Before getStats()");
getStats(item, startDate, endDate);
logger.error("After getStats()");
Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, context, count);
data.add(harvest);
logger.debug(harvest.toString());
return data;
} catch (Exception e) {
logger.error("Error Harvesting Data Methods Download for context {}", context, e);
throw e;
}
return data;
}

3
src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java → src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest.harvester;
package org.gcube.dataharvest.harvester.sobigdata;
import java.text.ParseException;
import java.util.ArrayList;
@ -18,6 +18,7 @@ import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFact
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

3
src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java → src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataharvest.harvester;
package org.gcube.dataharvest.harvester.sobigdata;
import java.text.ParseException;
import java.util.ArrayList;
@ -7,6 +7,7 @@ import java.util.Iterator;
import java.util.List;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
import org.json.JSONArray;

8
src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java
View File

@ -9,7 +9,8 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
import java.util.SortedSet;
import java.util.TreeSet;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.provider.UserInfo;
@ -49,7 +50,7 @@ public class ContextAuthorization {
* Contains Properties used to generate tokens
*/
public ContextAuthorization() throws Exception {
this.contextToToken = new TreeMap<>();
this.contextToToken = new HashMap<>();
this.tokenToContext = new HashMap<>();
retrieveContextsAndTokens();
}
@ -116,6 +117,9 @@ public class ContextAuthorization {
return tokenToContext.get(token);
}
public SortedSet<String> getContexts(){
return new TreeSet<String>(contextToToken.keySet());
}

23
src/main/java/org/gcube/dataharvest/utils/Utils.java
View File

@ -11,6 +11,11 @@ import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -44,5 +49,21 @@ public class Utils {
}
return json;
}
public static String getCurrentContext() throws ObjectNotFound, Exception{
return getCurrentContext(SecurityTokenProvider.instance.get());
}
public static String getCurrentContext(String token) throws ObjectNotFound, Exception{
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String context = authorizationEntry.getContext();
logger.info("Context of token {} is {}", token, context);
return context;
}
public static void setContext(String token) throws ObjectNotFound, Exception{
SecurityTokenProvider.instance.set(token);
ScopeProvider.instance.set(getCurrentContext(token));
}
}

6
src/test/java/org/gcube/dataharvest/Harvester.java
View File

@ -11,11 +11,11 @@ import org.gcube.dataharvest.dao.DatabaseConnectionData;
import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.harvester.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.ResourceCatalogueHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VreUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
import org.gcube.dataharvest.harvester.sobigdata.MethodInvocationHarvester;
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

15
src/test/java/org/gcube/dataharvest/utils/ScopedTest.java
View File

@ -76,22 +76,9 @@ public class ScopedTest {
DEFAULT_TEST_SCOPE = GCUBE;
}
public static String getCurrentScope(String token) throws ObjectNotFound, Exception{
AuthorizationEntry authorizationEntry = Constants.authorizationService().get(token);
String context = authorizationEntry.getContext();
logger.info("Context of token {} is {}", token, context);
return context;
}
public static void setContext(String token) throws ObjectNotFound, Exception{
SecurityTokenProvider.instance.set(token);
ScopeProvider.instance.set(getCurrentScope(token));
}
@BeforeClass
public static void beforeClass() throws Exception{
setContext(DEFAULT_TEST_SCOPE);
Utils.setContext(DEFAULT_TEST_SCOPE);
}
@AfterClass