gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167586 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Luca Frosini 2018-05-18 10:00:35 +00:00
parent 0f3b397fbd
commit a91ece8a0f
11 changed files with 69 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -10,7 +10,7 @@ import java.util.Properties;
import java.util.SortedSet;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VreUsersHarvester;
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
@ -133,7 +133,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
SortedSet<String> contexts = contextAuthorization.getContexts();
ArrayList<Harvest> data = new ArrayList<Harvest>();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
for(String context : contexts) {
@ -143,7 +143,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on VRE users
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
List<Harvest> harvested = vreUsersHarvester.getData();
List<HarvestedData> harvested = vreUsersHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
@ -153,7 +153,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<Harvest> harvested = resourceCatalogueHarvester.getData();
List<HarvestedData> harvested = resourceCatalogueHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
@ -162,7 +162,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
List<Harvest> harvested = dataMethodDownloadHarvester.getData();
List<HarvestedData> harvested = dataMethodDownloadHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
@ -171,7 +171,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(start, end);
List<Harvest> harvested = socialHarvester.getData();
List<HarvestedData> harvested = socialHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
@ -180,7 +180,7 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<Harvest> harvested = methodInvocationHarvester.getData();
List<HarvestedData> harvested = methodInvocationHarvester.getData();
data.addAll(harvested);
} catch(Exception x) {
logger.error(x.getLocalizedMessage());

6
src/main/java/org/gcube/dataharvest/dao/Dao.java
View File

@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.Utils;
import org.postgresql.util.PSQLException;
import org.slf4j.Logger;
@ -136,7 +136,7 @@ public class Dao {
* @param to
* @throws DaoException
*/
public void insertMonthlyMeasure(List<Harvest> data, java.util.Date from, java.util.Date to, boolean doUpdate)
public void insertMonthlyMeasure(List<HarvestedData> data, java.util.Date from, java.util.Date to, boolean doUpdate)
throws DaoException {
// first of all: check if data of the same type are already in the
// database.
@ -162,7 +162,7 @@ public class Dao {
monthFrom++; // because january = 0...
try {
for (Harvest harvest : data) {
for (HarvestedData harvest : data) {
String query = "select id from monthly_measure where measure_type_id=" + harvest.getDataType()
+ " and context_id=(select id from context where dname='" + harvest.getContext()
+ "') and month=" + monthFrom + " and year=" + yearFrom;

4
src/main/java/org/gcube/dataharvest/dao/DatabaseManager.java
View File

@ -3,7 +3,7 @@ package org.gcube.dataharvest.dao;
import java.util.Date;
import java.util.List;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -15,7 +15,7 @@ public class DatabaseManager {
}
public void insertMonthlyData(Date from, Date to, List<Harvest> data, boolean updateFlag) {
public void insertMonthlyData(Date from, Date to, List<HarvestedData> data, boolean updateFlag) {
Dao dao = null;
try {
dao = dbConnect();

8
src/main/java/org/gcube/dataharvest/datamodel/Harvest.java → src/main/java/org/gcube/dataharvest/datamodel/HarvestedData.java
View File

@ -3,7 +3,7 @@ package org.gcube.dataharvest.datamodel;
import java.util.Date;
import java.io.Serializable;
public class Harvest implements Serializable {
public class HarvestedData implements Serializable {
/**
* Generated Serial Version UID
@ -28,11 +28,11 @@ public class Harvest implements Serializable {
private long measure;
private Date day;
public Harvest() {
public HarvestedData() {
}
public Harvest(int dataType, String context, long measure, Date day) {
public HarvestedData(int dataType, String context, long measure, Date day) {
super();
this.dataType = dataType;
this.context = context;
@ -40,7 +40,7 @@ public class Harvest implements Serializable {
this.day = day;
}
public Harvest(int dataType, String context, long measure) {
public HarvestedData(int dataType, String context, long measure) {
this.dataType = dataType;
this.context = context;
this.measure = measure;

4
src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java
View File

@ -8,7 +8,7 @@ import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -44,6 +44,6 @@ public abstract class BasicHarvester {
return getCurrentContext(token);
}
public abstract List<Harvest> getData() throws Exception;
public abstract List<HarvestedData> getData() throws Exception;
}

12
src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java
View File

@ -8,7 +8,7 @@ import java.util.Date;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONArray;
import org.json.JSONObject;
@ -29,25 +29,25 @@ public class SocialHarvester extends BasicHarvester {
}
@Override
public List<Harvest> getData() throws Exception {
public List<HarvestedData> getData() throws Exception {
String context = Utils.getCurrentContext();
try {
ArrayList<Harvest> data = new ArrayList<Harvest>();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
getJson();
Harvest likesH = new Harvest(Harvest.SOCIAL_LIKES, context, likes);
HarvestedData likesH = new HarvestedData(HarvestedData.SOCIAL_LIKES, context, likes);
logger.debug("{}", likesH);
data.add(likesH);
Harvest postsH = new Harvest(Harvest.SOCIAL_POSTS, context, posts);
HarvestedData postsH = new HarvestedData(HarvestedData.SOCIAL_POSTS, context, posts);
logger.debug("{}", postsH);
data.add(postsH);
Harvest socialReplies = new Harvest(Harvest.SOCIAL_REPLIES, context, replies);
HarvestedData socialReplies = new HarvestedData(HarvestedData.SOCIAL_REPLIES, context, replies);
logger.debug("{}", socialReplies);
data.add(socialReplies);

8
src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java
View File

@ -8,7 +8,7 @@ import java.util.Date;
import java.util.List;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.utils.Utils;
import org.json.JSONObject;
import org.slf4j.Logger;
@ -24,13 +24,13 @@ public class VreUsersHarvester extends BasicHarvester {
}
@Override
public List<Harvest> getData() throws Exception {
public List<HarvestedData> getData() throws Exception {
String context = Utils.getCurrentContext();
try {
int measure = get();
Harvest harvest = new Harvest(Harvest.USERS, context, measure);
HarvestedData harvest = new HarvestedData(HarvestedData.USERS, context, measure);
logger.debug(harvest.toString());
ArrayList<Harvest> data = new ArrayList<Harvest>();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
data.add(harvest);
return data;
} catch (Exception e) {

8
src/main/java/org/gcube/dataharvest/harvester/sobigdata/DataMethodDownloadHarvester.java
View File

@ -16,7 +16,7 @@ import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.utils.Utils;
import org.slf4j.Logger;
@ -33,11 +33,11 @@ public class DataMethodDownloadHarvester extends BasicHarvester {
}
@Override
public List<Harvest> getData() throws Exception {
public List<HarvestedData> getData() throws Exception {
String context = Utils.getCurrentContext();
try {
ArrayList<Harvest> data = new ArrayList<Harvest>();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
count = 0;
@ -57,7 +57,7 @@ public class DataMethodDownloadHarvester extends BasicHarvester {
logger.error("Before getStats()");
getStats(item, startDate, endDate);
logger.error("After getStats()");
Harvest harvest = new Harvest(Harvest.DATA_METHOD_DOWNLOAD, context, count);
HarvestedData harvest = new HarvestedData(HarvestedData.DATA_METHOD_DOWNLOAD, context, count);
data.add(harvest);
logger.debug(harvest.toString());
return data;

6
src/main/java/org/gcube/dataharvest/harvester/sobigdata/MethodInvocationHarvester.java
View File

@ -17,7 +17,7 @@ import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -31,10 +31,10 @@ public class MethodInvocationHarvester extends BasicHarvester {
}
@Override
public List<Harvest> getData() throws Exception {
public List<HarvestedData> getData() throws Exception {
try {
logger.debug("MethodInvocationHarvester::getData()");
ArrayList<Harvest> data = new ArrayList<Harvest>();
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
AccountingPersistenceQuery accountingPersistenceQuery = AccountingPersistenceQueryFactory.getInstance();
Calendar start = Calendar.getInstance();
Calendar end = Calendar.getInstance();

52
src/main/java/org/gcube/dataharvest/harvester/sobigdata/ResourceCatalogueHarvester.java
View File

@ -6,7 +6,7 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
@ -30,54 +30,54 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
}
@Override
public List<Harvest> getData() throws Exception {
public List<HarvestedData> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
List<Harvest> dataDeliverable = getDataDeliverable();
for(Harvest harvest : dataDeliverable) {
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
List<HarvestedData> dataDeliverable = getDataDeliverable();
for(HarvestedData harvest : dataDeliverable) {
data.add(harvest);
}
List<Harvest> dataMethod = getDataMethod();
for(Harvest harvest : dataMethod) {
List<HarvestedData> dataMethod = getDataMethod();
for(HarvestedData harvest : dataMethod) {
data.add(harvest);
}
List<Harvest> dataDataset = getDataDataset();
for(Harvest harvest : dataDataset) {
List<HarvestedData> dataDataset = getDataDataset();
for(HarvestedData harvest : dataDataset) {
data.add(harvest);
}
List<Harvest> dataApplication = getDataApplication();
for(Harvest harvest : dataApplication) {
List<HarvestedData> dataApplication = getDataApplication();
for(HarvestedData harvest : dataApplication) {
data.add(harvest);
}
return data;
}
public List<Harvest> getDataDeliverable() throws Exception {
public List<HarvestedData> getDataDeliverable() throws Exception {
String json = executeQuery("Deliverable");
return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES);
return buildList(json, HarvestedData.NEW_CATALOGUE_DELIVERABLES);
}
public List<Harvest> getDataMethod() throws Exception {
public List<HarvestedData> getDataMethod() throws Exception {
String json = executeQuery("Method");
return buildList(json, Harvest.NEW_CATALOGUE_METHODS);
return buildList(json, HarvestedData.NEW_CATALOGUE_METHODS);
}
public List<Harvest> getDataDataset() throws Exception {
public List<HarvestedData> getDataDataset() throws Exception {
String json = executeQuery("Dataset");
return buildList(json, Harvest.NEW_CATALOGUE_DATASETS);
return buildList(json, HarvestedData.NEW_CATALOGUE_DATASETS);
}
public List<Harvest> getDataApplication() throws Exception {
public List<HarvestedData> getDataApplication() throws Exception {
String json = executeQuery("Application");
return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS);
return buildList(json, HarvestedData.NEW_CATALOGUE_APPLICATIONS);
}
private List<Harvest> buildList(String json, int dataType) throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
private List<HarvestedData> buildList(String json, int dataType) throws Exception {
ArrayList<HarvestedData> data = new ArrayList<HarvestedData>();
JSONObject jsonObject = new JSONObject(json);
JSONObject responseHeader = jsonObject.getJSONObject("responseHeader");
@ -91,7 +91,7 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
JSONObject response = jsonObject.getJSONObject("response");
int numFound = response.getInt("numFound");
Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
HarvestedData h = new HarvestedData(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
numFound);
logger.debug(h.toString());
data.add(h);
@ -112,18 +112,18 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
}
}
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
h = new HarvestedData(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
cityOfCitizensCounter);
logger.debug(h.toString());
data.add(h);
h = new Harvest(dataType, "/Migration Studies", migrationStudiesCounter);
h = new HarvestedData(dataType, "/Migration Studies", migrationStudiesCounter);
logger.debug(h.toString());
data.add(h);
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates",
h = new HarvestedData(dataType, "/d4science.research-infrastructures.eu/SoBigData/SocietalDebates",
societalDebatesCounter);
logger.debug(h.toString());
data.add(h);
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy",
h = new HarvestedData(dataType, "/d4science.research-infrastructures.eu/SoBigData/WellBeingAndEconomy",
wellBeingAndEconomyCounter);
logger.debug(h.toString());
data.add(h);

16
src/test/java/org/gcube/dataharvest/Harvester.java
View File

@ -9,7 +9,7 @@ import org.gcube.dataharvest.dao.Dao;
import org.gcube.dataharvest.dao.DaoException;
import org.gcube.dataharvest.dao.DatabaseConnectionData;
import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.datamodel.HarvestedData;
import org.gcube.dataharvest.harvester.BasicHarvester;
import org.gcube.dataharvest.harvester.SocialHarvester;
import org.gcube.dataharvest.harvester.VreUsersHarvester;
@ -70,7 +70,7 @@ public class Harvester {
public void runOne(BasicHarvester harvester) {
try {
List<Harvest> data = harvester.getData();
List<HarvestedData> data = harvester.getData();
if (data != null) {
insertMonthlyData((Date) dateFrom, (Date) dateTo, data);
}
@ -83,7 +83,7 @@ public class Harvester {
try {
// collecting info on VRE users
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(dateFrom, dateTo);
List<Harvest> users = vreUsersHarvester.getData();
List<HarvestedData> users = vreUsersHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, users);
} catch (Exception x) {
logger.error(x.getLocalizedMessage());
@ -93,7 +93,7 @@ public class Harvester {
// collecting info on Res. Catalogue (Dataset, Application,
// Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
List<Harvest> res = resourceCatalogueHarvester.getData();
List<HarvestedData> res = resourceCatalogueHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch (Exception x) {
logger.error(x.getLocalizedMessage());
@ -102,7 +102,7 @@ public class Harvester {
try {
// collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
List<Harvest> res = dataMethodDownloadHarvester.getData();
List<HarvestedData> res = dataMethodDownloadHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch (Exception x) {
logger.error(x.getLocalizedMessage());
@ -111,7 +111,7 @@ public class Harvester {
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo);
List<Harvest> res = socialHarvester.getData();
List<HarvestedData> res = socialHarvester.getData();
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch (Exception x) {
logger.error(x.getLocalizedMessage());
@ -120,7 +120,7 @@ public class Harvester {
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(dateFrom, dateTo);
List<Harvest> res = methodInvocationHarvester.getData();
List<HarvestedData> res = methodInvocationHarvester.getData();
logger.debug("{}", res);
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
} catch (Exception x) {
@ -128,7 +128,7 @@ public class Harvester {
}
}
private void insertMonthlyData(Date from, Date to, List<Harvest> data) {
private void insertMonthlyData(Date from, Date to, List<HarvestedData> data) {
Dao dao = null;
try {
dao = dbConnect();