gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167568 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-17 15:21:08 +00:00
parent 9dec451f4c
commit 8d64022afa
10 changed files with 162 additions and 437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -72,28 +72,11 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
};
public static InheritableThreadLocal<ContextAuthorization> getContextAuthorization() {
return contextAuthorization;
}
private void retrieveAuthorizations() throws Exception {
ContextAuthorization contextAuthorization = new ContextAuthorization();
getContextAuthorization().set(contextAuthorization);
}
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
super(pluginDeclaration);
}
/** {@inheritDoc} */
@Override
public void launch(Map<String,Object> inputs) throws Exception {
logger.debug("{} is starting", this.getClass().getSimpleName());
getConfigParameters();
retrieveAuthorizations();
if(inputs == null || inputs.isEmpty()) {
throw new IllegalArgumentException("The can only be launched providing valid input parameters");
}
@ -134,63 +117,71 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.debug("Harvesting from {} to {}", DateUtils.LAUNCH_DATE_FORMAT.format(start),
DateUtils.LAUNCH_DATE_FORMAT.format(end));
getConfigParameters();
ContextAuthorization contextAuthorization = new ContextAuthorization();
DatabaseManager dbaseManager = new DatabaseManager();
try {
// collecting info on VRE users
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
List<Harvest> users = vreUsersHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, users, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<Harvest> res = resourceCatalogueHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
for() {
try {
// collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
List<Harvest> res = dataMethodDownloadHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
try {
// collecting info on VRE users
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
List<Harvest> users = vreUsersHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, users, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(start, end);
List<Harvest> res = socialHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
try {
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<Harvest> res = resourceCatalogueHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<Harvest> res = methodInvocationHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
try {
// collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
List<Harvest> res = dataMethodDownloadHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(start, end);
List<Harvest> res = socialHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<Harvest> res = methodInvocationHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
}
/** {@inheritDoc} */

6
src/main/java/org/gcube/dataharvest/datamodel/Harvest.java
View File

@ -4,10 +4,12 @@ import java.util.Date;
import java.io.Serializable;
public class Harvest implements Serializable {
/**
*
* Generated Serial Version UID
*/
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 3699669951917080213L;
public static int ACCESSESS = 1;
public static int USERS = 2;
public static int DATA_METHOD_DOWNLOAD = 3;

109
src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java
View File

@ -1,47 +1,28 @@
package org.gcube.dataharvest.harvester;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.resources.gcore.utils.Group;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.informationsystem.publisher.RegistryPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class);
public Date startDate = null, endDate = null;
public DateFormat format;
public Date startDate;
public Date endDate;
public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester";
public final String CATEGORY_NAME = "Accounting";
public BasicHarvester(String start, String end) throws ParseException {
format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
startDate = format.parse(start);
endDate = format.parse(end);
}
public BasicHarvester(Date start, Date end) throws ParseException {
startDate = start;
endDate = end;
@ -69,86 +50,6 @@ public class BasicHarvester {
return null;
}
public Properties readServiceEndpoint() throws Exception {
Properties props = new Properties();
String scope = "/d4science.research-infrastructures.eu";
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
String password = null;
try {
List<ServiceEndpoint> list = client.submit(query);
if (list.size() > 1) {
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
+ " in this scope having Category " + CATEGORY_NAME);
} else if (list.size() == 0) {
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
} else {
for (ServiceEndpoint res : list) {
AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
.toArray(new AccessPoint[res.profile().accessPoints().size()]);
for (AccessPoint found : accessPoints) {
password = org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
for (ServiceEndpoint.Property prop : found.properties()) {
props.setProperty(prop.name(), org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
}
}
}
}
return props;
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
}
public void updateServiceEndPoint(ArrayList<Property> newProps) throws Exception {
String scope = "/d4science.research-infrastructures.eu";
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
String password = null;
try {
List<ServiceEndpoint> list = client.submit(query);
if (list.size() > 1) {
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
+ " in this scope having Category " + CATEGORY_NAME);
} else if (list.size() == 0) {
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
} else {
ServiceEndpoint serviceEndpoint = list.get(0);
AccessPoint accessPoint = ((AccessPoint[]) serviceEndpoint.profile().accessPoints().toArray(new AccessPoint[serviceEndpoint.profile().accessPoints().size()]))[0];
Group<Property> oldProps = accessPoint.properties();
for (Property prop : newProps) {
String propValue = prop.value();
propValue = org.gcube.common.encryption.StringEncrypter.getEncrypter().encrypt(propValue);
String propKey = prop.name();
org.gcube.common.resources.gcore.ServiceEndpoint.Property pToAdd =
new org.gcube.common.resources.gcore.ServiceEndpoint.Property().nameAndValue(propKey, propValue);
pToAdd.encrypted(true);
accessPoint.properties().add(pToAdd);
}
RegistryPublisher publisher = RegistryPublisherFactory.create();
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
publisher.update(serviceEndpoint);
}
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
}
public String[] getActiveVREs(boolean testMode) {
DatabaseManager dbaseManager = new DatabaseManager();
return dbaseManager.getActiveVres();

47
src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java
View File

@ -1,81 +1,58 @@
package org.gcube.dataharvest.harvester;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang.Validate;
import org.gcube.common.homelibrary.home.Home;
import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.HomeManager;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongDestinationException;
import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
import org.gcube.common.homelibrary.model.exceptions.RepositoryException;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataMethodDownloadHarvester extends BasicHarvester {
private Date startDate, endDate;
private DateFormat format;
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class);
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
private int count = 0;
public DataMethodDownloadHarvester(String start, String end) throws ParseException {
super(start, end);
//ScopeProvider.instance.set(SCOPE);
}
public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
super(start, end);
//ScopeProvider.instance.set(SCOPE);
}
@Override
public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
ScopeProvider.instance.set(SCOPE);
String[] vres = getActiveVREs(true);
for (String vre : vres) {
try {
count = 0;
logger.error("HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager()");
HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager();
String user = getVREName(vre) + "-Manager";
logger.error("Home home = manager.getHome(user)");
Home home = manager.getHome(user);
logger.error("JCRWorkspace ws = (JCRWorkspace) home.getWorkspace()");
JCRWorkspace ws = (JCRWorkspace) home.getWorkspace();
logger.error("ws.getItemByPath...");
JCRWorkspaceItem item = (JCRWorkspaceItem) ws
.getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre));

48
src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java
View File

@ -3,25 +3,20 @@ package org.gcube.dataharvest.harvester;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import java.util.SortedMap;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.Iterator;
import org.gcube.dataharvest.datamodel.Harvest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -30,10 +25,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
public MethodInvocationHarvester(String start, String end) throws ParseException {
super(start, end);
}
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
@ -83,33 +74,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
logger.error(errorMessage);
throw x;
}
}/**/
/* used only for test */
public static void main(String[] args) {
String ROOT = "18fed2d9-030b-4c77-93af-af2015d945f7-843339462";
String GCUBE = "32542bb3-b04b-4112-9af1-7fac049d33cd-98187548";
String TAGME = "308cf6bd-3cbb-4267-a300-0cf5ae122bc9-843339462";
String TAGME2 = "9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462";
String SOBIGDATA_EU = "6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462";
try {
MethodInvocationHarvester m = new MethodInvocationHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59");
MethodInvocationHarvester.setContext(SOBIGDATA_EU);
System.out.println(MethodInvocationHarvester.getCurrentContext());
Properties props = m.readServiceEndpoint();
System.out.println("Found " + props.size() + " properties.");
Set keys = props.keySet();
for(Object key : keys) {
System.out.println((String)key + " :: " + props.getProperty((String) key));
}
m.getData();
} catch (Exception e) {
// TODO Auto-generated catch block
System.out.println(e.getLocalizedMessage());
e.printStackTrace();
}
}
/**/
}

137
src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java
View File

@ -1,127 +1,116 @@
package org.gcube.dataharvest.harvester;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Iterator;
import java.util.Locale;
import java.util.Properties;
import java.util.List;
import org.apache.http.client.utils.URLEncodedUtils;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.quartz.utils.StringKeyDirtyFlagMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ResourceCatalogueHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class);
private int cityOfCitizensCounter = 0, migrationStudiesCounter = 0, societalDebatesCounter = 0,
wellBeingAndEconomyCounter = 0;
public ResourceCatalogueHarvester(String start, String end) throws ParseException {
super(start, end);
}
private int cityOfCitizensCounter = 0;
private int migrationStudiesCounter = 0;
private int societalDebatesCounter = 0;
private int wellBeingAndEconomyCounter = 0;
public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
@Override
public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
List<Harvest> dataDeliverable = getDataDeliverable();
for (Harvest harvest : dataDeliverable) {
for(Harvest harvest : dataDeliverable) {
data.add(harvest);
}
List<Harvest> dataMethod = getDataMethod();
for (Harvest harvest : dataMethod) {
for(Harvest harvest : dataMethod) {
data.add(harvest);
}
List<Harvest> dataDataset = getDataDataset();
for (Harvest harvest : dataDataset) {
for(Harvest harvest : dataDataset) {
data.add(harvest);
}
List<Harvest> dataApplication = getDataApplication();
for (Harvest harvest : dataApplication) {
for(Harvest harvest : dataApplication) {
data.add(harvest);
}
return data;
}
public List<Harvest> getDataDeliverable() throws Exception {
String json = executeQuery("Deliverable");
return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES);
}
public List<Harvest> getDataMethod() throws Exception {
String json = executeQuery("Method");
return buildList(json, Harvest.NEW_CATALOGUE_METHODS);
}
public List<Harvest> getDataDataset() throws Exception {
String json = executeQuery("Dataset");
return buildList(json, Harvest.NEW_CATALOGUE_DATASETS);
}
public List<Harvest> getDataApplication() throws Exception {
String json = executeQuery("Application");
return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS);
}
private List<Harvest> buildList(String json, int dataType) throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
JSONObject jsonObject = new JSONObject(json);
JSONObject responseHeader = jsonObject.getJSONObject("responseHeader");
int status = responseHeader.getInt("status");
if (status != 0) {
if(status != 0) {
String err = "Query Deliverable in error: status " + status;
logger.error(err);
throw new Exception(err, null);
}
JSONObject response = jsonObject.getJSONObject("response");
int numFound = response.getInt("numFound");
Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
numFound);
logger.debug(h.toString());
data.add(h);
if (numFound > 0) {
if(numFound > 0) {
JSONArray docs = response.getJSONArray("docs");
for (Object item : docs) {
for(Object item : docs) {
JSONObject doc = (JSONObject) item;
try {
JSONArray groups = doc.getJSONArray("groups");
Iterator<Object> git = groups.iterator();
while (git.hasNext()) {
while(git.hasNext()) {
String groupItem = (String) git.next();
counterByGroup(groupItem);
}
} catch (JSONException x) {
} catch(JSONException x) {
logger.debug("Document without groups");
}
}
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
cityOfCitizensCounter);
logger.debug(h.toString());
@ -137,13 +126,13 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
wellBeingAndEconomyCounter);
logger.debug(h.toString());
data.add(h);
}
return data;
}
private String executeQuery(String fqSubString) throws Exception {
String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?";
String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO "
@ -152,49 +141,29 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\"");
query += "&fq=" + fq + "&wt=json&indent=true";
logger.debug(query);
String json = Utils.getJson(query);
// logger.debug(json);
return json;
}
private void counterByGroup(String groupName) {
cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0;
switch (groupName) {
case "city-of-citizens-group":
cityOfCitizensCounter++;
break;
case "migration-studies":
migrationStudiesCounter++;
break;
case "societal-debates-group":
societalDebatesCounter++;
break;
case "well-being-and-economy-group":
wellBeingAndEconomyCounter++;
break;
switch(groupName) {
case "city-of-citizens-group":
cityOfCitizensCounter++;
break;
case "migration-studies":
migrationStudiesCounter++;
break;
case "societal-debates-group":
societalDebatesCounter++;
break;
case "well-being-and-economy-group":
wellBeingAndEconomyCounter++;
break;
}
}
/*
* used only for test public static void main(String[] args) { Calendar from
* = Calendar.getInstance(); from.set(Calendar.DAY_OF_MONTH, 1);
* from.set(Calendar.MONTH, Calendar.NOVEMBER); from.set(Calendar.YEAR,
* 2017); from.set(Calendar.HOUR_OF_DAY, 0); from.set(Calendar.MINUTE, 0);
* from.set(Calendar.SECOND, 0);
*
* Calendar to = Calendar.getInstance(); to.set(Calendar.DAY_OF_MONTH, 30);
* to.set(Calendar.MONTH, Calendar.NOVEMBER); to.set(Calendar.YEAR, 2017);
* to.set(Calendar.HOUR_OF_DAY, 23); to.set(Calendar.MINUTE, 59);
* to.set(Calendar.SECOND, 59);
*
* try { ResourceCatalogueHarvester a = new
* ResourceCatalogueHarvester(from.getTime(), to.getTime()); List<Harvest> l
* = a.getData(); for(Harvest h : l) { System.out.println(h.toString()); }
* // System.out.println(Utils.dateToString(a.startDate)); //
* System.out.println(Utils.dateToStringWithTZ(a.endDate)); } catch
* (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); }
* }
*/
}

51
src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java
View File

@ -16,32 +16,26 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SocialHarvester extends BasicHarvester {
private final String CATEGORY_NAME = "Accounting";
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
private static Logger logger = LoggerFactory.getLogger(SocialHarvester.class);
private int likes, replies, posts;
private long from = 0, to = 0;
public SocialHarvester(String start, String end) throws ParseException {
super(start, end);
this.from = startDate.getTime();
this.to = endDate.getTime();
}
public SocialHarvester(Date start, Date end) throws ParseException {
super(start, end);
this.from = startDate.getTime();
this.to = endDate.getTime();
}
@Override
public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>();
Properties props = readServiceEndpoint();
String[] vres = getActiveVREs(true);
for (String vre : vres) {
for(String vre : vres) {
try {
logger.debug("Working on VRE: " + vre);
String key = vre; // vre.substring(vre.lastIndexOf("/") + 1);
String token = props.getProperty(key);
//System.out.println(key + ":" + token);
@ -50,7 +44,7 @@ public class SocialHarvester extends BasicHarvester {
Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes);
data.add(harvest);
logger.debug(harvest.toString());
harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts);
data.add(harvest);
logger.debug(harvest.toString());
@ -58,52 +52,41 @@ public class SocialHarvester extends BasicHarvester {
harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies);
data.add(harvest);
logger.debug(harvest.toString());
} catch (Exception x) {
} catch(Exception x) {
logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage());
}
}
return data;
}
private void getJson(String token) throws MalformedURLException, IOException {
likes = replies = posts = 0;
// la seguente stringa deve essere letta dinamicamente
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token=";
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
Boolean success = (Boolean) jsonObject.get("success");
if (success == false) {
if(success == false) {
String message = "getJson() on token: " + token + " success=false";
logger.error(message);
throw new IOException(message);
}
JSONArray res = jsonObject.getJSONArray("result");
int len = res.length();
for (int i = 0; i < len; i++) {
for(int i = 0; i < len; i++) {
JSONObject item = res.getJSONObject(i);
long time = item.getLong("time");
//System.out.println(from + " - " + time + " - " + to );
if ((from <= time) && (time <= to)) {
if((from <= time) && (time <= to)) {
posts++;
replies += item.getInt("comments_no");
likes += item.getInt("likes_no");
}
}
}
/**/
public static void main(String[] argv) {
try {
SocialHarvester a = new SocialHarvester("2018-01-01 00:00:00", "2018-01-31 23:59:59");
List<Harvest> list = a.getData();
for (Harvest l : list) {
System.out.println(l.toString());
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}/**/
}

67
src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java
View File

@ -15,13 +15,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class VreUsersHarvester extends BasicHarvester {
// private final String CATEGORY_NAME = "Accounting";
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
public VreUsersHarvester(String start, String end) throws ParseException {
super(start, end);
}
public VreUsersHarvester(Date start, Date end) throws ParseException {
super(start, end);
}
@ -60,63 +57,5 @@ public class VreUsersHarvester extends BasicHarvester {
userNumber = jsonObject.getJSONArray("result").length();
return userNumber;
}
// private Properties readServiceEndpoint() throws Exception {
// Properties props = new Properties();
// String scope = "/d4science.research-infrastructures.eu";
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
//
// SimpleQuery query = queryFor(ServiceEndpoint.class);
// query.addCondition("$resource/Profile/Name/text() eq '" +
// RUNTIME_RESOURCE_NAME + "'");
// query.addCondition("$resource/Profile/Category/text() eq '" +
// CATEGORY_NAME + "'");
//
// DiscoveryClient<ServiceEndpoint> client =
// clientFor(ServiceEndpoint.class);
// String password = null;
// try {
// List<ServiceEndpoint> list = client.submit(query);
// if (list.size() > 1) {
// logger.error("Too many Service Endpoints having name " +
// RUNTIME_RESOURCE_NAME
// + " in this scope having Category " + CATEGORY_NAME);
// } else if (list.size() == 0) {
// logger.error("There is no Service Endpoint having name " +
// RUNTIME_RESOURCE_NAME
// + " and Category " + CATEGORY_NAME + " in this scope " + scope);
// } else {
// for (ServiceEndpoint res : list) {
// AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
// .toArray(new AccessPoint[res.profile().accessPoints().size()]);
// for (AccessPoint found : accessPoints) {
// password =
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
// for (ServiceEndpoint.Property prop : found.properties()) {
// props.setProperty(prop.name(),
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
// }
// }
//
// }
// }
// return props;
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage());
// throw e;
// }
// }
// public static void main(String[] argv) {
// try {
// VreUsersHarvester a = new VreUsersHarvester("2017-01-01", "2017-02-01");
// List<Harvest> list = a.getData();
// for(Harvest l : list) {
// System.out.println(l.toString());
// }
// } catch (Exception e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// }
}

7
src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java
View File

@ -9,6 +9,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.provider.UserInfo;
@ -41,12 +42,14 @@ public class ContextAuthorization {
*/
protected Map<String, String> tokenToContext;
/**
* Contains Properties used to generate tokens
*/
public ContextAuthorization() throws Exception {
this.contextToToken = new HashMap<>();
this.contextToToken = new TreeMap<>();
this.tokenToContext = new HashMap<>();
retrieveContextsAndTokens();
}

4
src/main/java/org/gcube/dataharvest/utils/Utils.java
View File

@ -25,10 +25,6 @@ public class Utils {
return cal;
}
public static String dateToString(Date date) {
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
return formatter.format(date);
}
public static String dateToStringWithTZ(Date date) {
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");