gCubeSystem
/
accounting-dashboard-harvester-se-plugin
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code redesign 

Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor 

Task-Url: https://support.d4science.org/issues/11756

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167568 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2018-05-17 15:21:08 +00:00
parent 9dec451f4c
commit 8d64022afa
10 changed files with 162 additions and 437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java
View File

@ -72,28 +72,11 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
}; };
public static InheritableThreadLocal<ContextAuthorization> getContextAuthorization() {
return contextAuthorization;
}
private void retrieveAuthorizations() throws Exception {
ContextAuthorization contextAuthorization = new ContextAuthorization();
getContextAuthorization().set(contextAuthorization);
}
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
super(pluginDeclaration);
}
/** {@inheritDoc} */ /** {@inheritDoc} */
@Override @Override
public void launch(Map<String,Object> inputs) throws Exception { public void launch(Map<String,Object> inputs) throws Exception {
logger.debug("{} is starting", this.getClass().getSimpleName()); logger.debug("{} is starting", this.getClass().getSimpleName());
getConfigParameters();
retrieveAuthorizations();
if(inputs == null || inputs.isEmpty()) { if(inputs == null || inputs.isEmpty()) {
throw new IllegalArgumentException("The can only be launched providing valid input parameters"); throw new IllegalArgumentException("The can only be launched providing valid input parameters");
} }
@ -134,63 +117,71 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
logger.debug("Harvesting from {} to {}", DateUtils.LAUNCH_DATE_FORMAT.format(start), logger.debug("Harvesting from {} to {}", DateUtils.LAUNCH_DATE_FORMAT.format(start),
DateUtils.LAUNCH_DATE_FORMAT.format(end)); DateUtils.LAUNCH_DATE_FORMAT.format(end));
getConfigParameters();
ContextAuthorization contextAuthorization = new ContextAuthorization();
DatabaseManager dbaseManager = new DatabaseManager(); DatabaseManager dbaseManager = new DatabaseManager();
try {
// collecting info on VRE users
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
List<Harvest> users = vreUsersHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, users, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try { for() {
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
List<Harvest> res = resourceCatalogueHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on Data/Method download try {
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end); // collecting info on VRE users
List<Harvest> res = dataMethodDownloadHarvester.getData(); VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
if(!dryRun) { List<Harvest> users = vreUsersHarvester.getData();
dbaseManager.insertMonthlyData(start, end, res, reRun); if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, users, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
} }
} catch(Exception x) {
logger.error(x.getLocalizedMessage()); try {
} // collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
try { List<Harvest> res = resourceCatalogueHarvester.getData();
// collecting info on social (posts, replies and likes) if(!dryRun) {
SocialHarvester socialHarvester = new SocialHarvester(start, end); dbaseManager.insertMonthlyData(start, end, res, reRun);
List<Harvest> res = socialHarvester.getData(); }
if(!dryRun) { } catch(Exception x) {
dbaseManager.insertMonthlyData(start, end, res, reRun); logger.error(x.getLocalizedMessage());
} }
} catch(Exception x) {
logger.error(x.getLocalizedMessage()); try {
} // collecting info on Data/Method download
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
try { List<Harvest> res = dataMethodDownloadHarvester.getData();
// collecting info on method invocation if(!dryRun) {
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); dbaseManager.insertMonthlyData(start, end, res, reRun);
List<Harvest> res = methodInvocationHarvester.getData(); }
if(!dryRun) { } catch(Exception x) {
dbaseManager.insertMonthlyData(start, end, res, reRun); logger.error(x.getLocalizedMessage());
}
try {
// collecting info on social (posts, replies and likes)
SocialHarvester socialHarvester = new SocialHarvester(start, end);
List<Harvest> res = socialHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
}
try {
// collecting info on method invocation
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
List<Harvest> res = methodInvocationHarvester.getData();
if(!dryRun) {
dbaseManager.insertMonthlyData(start, end, res, reRun);
}
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
} }
} catch(Exception x) {
logger.error(x.getLocalizedMessage());
} }
} }
/** {@inheritDoc} */ /** {@inheritDoc} */

6
src/main/java/org/gcube/dataharvest/datamodel/Harvest.java
View File

@ -4,10 +4,12 @@ import java.util.Date;
import java.io.Serializable; import java.io.Serializable;
public class Harvest implements Serializable { public class Harvest implements Serializable {
/** /**
* * Generated Serial Version UID
*/ */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 3699669951917080213L;
public static int ACCESSESS = 1; public static int ACCESSESS = 1;
public static int USERS = 2; public static int USERS = 2;
public static int DATA_METHOD_DOWNLOAD = 3; public static int DATA_METHOD_DOWNLOAD = 3;

109
src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java
View File

@ -1,47 +1,28 @@
package org.gcube.dataharvest.harvester; package org.gcube.dataharvest.harvester;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.text.DateFormat;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Properties;
import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.client.Constants;
import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
import org.gcube.common.resources.gcore.utils.Group;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.dao.DatabaseManager;
import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.informationsystem.publisher.RegistryPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class BasicHarvester { public class BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class); private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class);
public Date startDate = null, endDate = null;
public DateFormat format; public Date startDate;
public Date endDate;
public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester"; public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester";
public final String CATEGORY_NAME = "Accounting"; public final String CATEGORY_NAME = "Accounting";
public BasicHarvester(String start, String end) throws ParseException {
format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
startDate = format.parse(start);
endDate = format.parse(end);
}
public BasicHarvester(Date start, Date end) throws ParseException { public BasicHarvester(Date start, Date end) throws ParseException {
startDate = start; startDate = start;
endDate = end; endDate = end;
@ -69,86 +50,6 @@ public class BasicHarvester {
return null; return null;
} }
public Properties readServiceEndpoint() throws Exception {
Properties props = new Properties();
String scope = "/d4science.research-infrastructures.eu";
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
String password = null;
try {
List<ServiceEndpoint> list = client.submit(query);
if (list.size() > 1) {
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
+ " in this scope having Category " + CATEGORY_NAME);
} else if (list.size() == 0) {
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
} else {
for (ServiceEndpoint res : list) {
AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
.toArray(new AccessPoint[res.profile().accessPoints().size()]);
for (AccessPoint found : accessPoints) {
password = org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
for (ServiceEndpoint.Property prop : found.properties()) {
props.setProperty(prop.name(), org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
}
}
}
}
return props;
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
}
public void updateServiceEndPoint(ArrayList<Property> newProps) throws Exception {
String scope = "/d4science.research-infrastructures.eu";
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
String password = null;
try {
List<ServiceEndpoint> list = client.submit(query);
if (list.size() > 1) {
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
+ " in this scope having Category " + CATEGORY_NAME);
} else if (list.size() == 0) {
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
} else {
ServiceEndpoint serviceEndpoint = list.get(0);
AccessPoint accessPoint = ((AccessPoint[]) serviceEndpoint.profile().accessPoints().toArray(new AccessPoint[serviceEndpoint.profile().accessPoints().size()]))[0];
Group<Property> oldProps = accessPoint.properties();
for (Property prop : newProps) {
String propValue = prop.value();
propValue = org.gcube.common.encryption.StringEncrypter.getEncrypter().encrypt(propValue);
String propKey = prop.name();
org.gcube.common.resources.gcore.ServiceEndpoint.Property pToAdd =
new org.gcube.common.resources.gcore.ServiceEndpoint.Property().nameAndValue(propKey, propValue);
pToAdd.encrypted(true);
accessPoint.properties().add(pToAdd);
}
RegistryPublisher publisher = RegistryPublisherFactory.create();
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
publisher.update(serviceEndpoint);
}
} catch (Exception e) {
logger.error(e.getLocalizedMessage());
throw e;
}
}
public String[] getActiveVREs(boolean testMode) { public String[] getActiveVREs(boolean testMode) {
DatabaseManager dbaseManager = new DatabaseManager(); DatabaseManager dbaseManager = new DatabaseManager();
return dbaseManager.getActiveVres(); return dbaseManager.getActiveVres();

47
src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java
View File

@ -1,81 +1,58 @@
package org.gcube.dataharvest.harvester; package org.gcube.dataharvest.harvester;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.text.DateFormat;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang.Validate; import org.apache.commons.lang.Validate;
import org.gcube.common.homelibrary.home.Home; import org.gcube.common.homelibrary.home.Home;
import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.homelibrary.home.HomeLibrary;
import org.gcube.common.homelibrary.home.HomeManager; import org.gcube.common.homelibrary.home.HomeManager;
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongDestinationException;
import org.gcube.common.homelibrary.jcr.repository.JCRRepository; import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
import org.gcube.common.homelibrary.model.exceptions.RepositoryException;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataMethodDownloadHarvester extends BasicHarvester { public class DataMethodDownloadHarvester extends BasicHarvester {
private Date startDate, endDate;
private DateFormat format;
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class); private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class);
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
private int count = 0; private int count = 0;
public DataMethodDownloadHarvester(String start, String end) throws ParseException {
super(start, end);
//ScopeProvider.instance.set(SCOPE);
}
public DataMethodDownloadHarvester(Date start, Date end) throws ParseException { public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
super(start, end); super(start, end);
//ScopeProvider.instance.set(SCOPE);
} }
@Override @Override
public List<Harvest> getData() throws Exception { public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>(); ArrayList<Harvest> data = new ArrayList<Harvest>();
ScopeProvider.instance.set(SCOPE); ScopeProvider.instance.set(SCOPE);
String[] vres = getActiveVREs(true); String[] vres = getActiveVREs(true);
for (String vre : vres) { for (String vre : vres) {
try { try {
count = 0; count = 0;
logger.error("HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager()");
HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager(); HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager();
String user = getVREName(vre) + "-Manager"; String user = getVREName(vre) + "-Manager";
logger.error("Home home = manager.getHome(user)");
Home home = manager.getHome(user); Home home = manager.getHome(user);
logger.error("JCRWorkspace ws = (JCRWorkspace) home.getWorkspace()");
JCRWorkspace ws = (JCRWorkspace) home.getWorkspace(); JCRWorkspace ws = (JCRWorkspace) home.getWorkspace();
logger.error("ws.getItemByPath...");
JCRWorkspaceItem item = (JCRWorkspaceItem) ws JCRWorkspaceItem item = (JCRWorkspaceItem) ws
.getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre)); .getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre));

48
src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java
View File

@ -3,25 +3,20 @@ package org.gcube.dataharvest.harvester;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Filter;
import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.Info;
import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint;
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
import java.util.SortedMap;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.Iterator;
import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.datamodel.Harvest;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -30,10 +25,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class); private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
public MethodInvocationHarvester(String start, String end) throws ParseException {
super(start, end);
}
public MethodInvocationHarvester(Date start, Date end) throws ParseException { public MethodInvocationHarvester(Date start, Date end) throws ParseException {
super(start, end); super(start, end);
} }
@ -83,33 +74,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
logger.error(errorMessage); logger.error(errorMessage);
throw x; throw x;
} }
}/**/
/* used only for test */
public static void main(String[] args) {
String ROOT = "18fed2d9-030b-4c77-93af-af2015d945f7-843339462";
String GCUBE = "32542bb3-b04b-4112-9af1-7fac049d33cd-98187548";
String TAGME = "308cf6bd-3cbb-4267-a300-0cf5ae122bc9-843339462";
String TAGME2 = "9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462";
String SOBIGDATA_EU = "6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462";
try {
MethodInvocationHarvester m = new MethodInvocationHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59");
MethodInvocationHarvester.setContext(SOBIGDATA_EU);
System.out.println(MethodInvocationHarvester.getCurrentContext());
Properties props = m.readServiceEndpoint();
System.out.println("Found " + props.size() + " properties.");
Set keys = props.keySet();
for(Object key : keys) {
System.out.println((String)key + " :: " + props.getProperty((String) key));
}
m.getData();
} catch (Exception e) {
// TODO Auto-generated catch block
System.out.println(e.getLocalizedMessage());
e.printStackTrace();
}
} }
/**/
} }

137
src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java
View File

@ -1,127 +1,116 @@
package org.gcube.dataharvest.harvester; package org.gcube.dataharvest.harvester;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.List;
import java.util.Iterator; import java.util.Iterator;
import java.util.Locale; import java.util.List;
import java.util.Properties;
import org.apache.http.client.utils.URLEncodedUtils;
import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.datamodel.Harvest;
import org.gcube.dataharvest.utils.Utils; import org.gcube.dataharvest.utils.Utils;
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil; import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.quartz.utils.StringKeyDirtyFlagMap;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class ResourceCatalogueHarvester extends BasicHarvester { public class ResourceCatalogueHarvester extends BasicHarvester {
private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class); private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class);
private int cityOfCitizensCounter = 0, migrationStudiesCounter = 0, societalDebatesCounter = 0,
wellBeingAndEconomyCounter = 0; private int cityOfCitizensCounter = 0;
private int migrationStudiesCounter = 0;
public ResourceCatalogueHarvester(String start, String end) throws ParseException { private int societalDebatesCounter = 0;
super(start, end); private int wellBeingAndEconomyCounter = 0;
}
public ResourceCatalogueHarvester(Date start, Date end) throws ParseException { public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
super(start, end); super(start, end);
} }
@Override @Override
public List<Harvest> getData() throws Exception { public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>(); ArrayList<Harvest> data = new ArrayList<Harvest>();
List<Harvest> dataDeliverable = getDataDeliverable(); List<Harvest> dataDeliverable = getDataDeliverable();
for (Harvest harvest : dataDeliverable) { for(Harvest harvest : dataDeliverable) {
data.add(harvest); data.add(harvest);
} }
List<Harvest> dataMethod = getDataMethod(); List<Harvest> dataMethod = getDataMethod();
for (Harvest harvest : dataMethod) { for(Harvest harvest : dataMethod) {
data.add(harvest); data.add(harvest);
} }
List<Harvest> dataDataset = getDataDataset(); List<Harvest> dataDataset = getDataDataset();
for (Harvest harvest : dataDataset) { for(Harvest harvest : dataDataset) {
data.add(harvest); data.add(harvest);
} }
List<Harvest> dataApplication = getDataApplication(); List<Harvest> dataApplication = getDataApplication();
for (Harvest harvest : dataApplication) { for(Harvest harvest : dataApplication) {
data.add(harvest); data.add(harvest);
} }
return data; return data;
} }
public List<Harvest> getDataDeliverable() throws Exception { public List<Harvest> getDataDeliverable() throws Exception {
String json = executeQuery("Deliverable"); String json = executeQuery("Deliverable");
return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES); return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES);
} }
public List<Harvest> getDataMethod() throws Exception { public List<Harvest> getDataMethod() throws Exception {
String json = executeQuery("Method"); String json = executeQuery("Method");
return buildList(json, Harvest.NEW_CATALOGUE_METHODS); return buildList(json, Harvest.NEW_CATALOGUE_METHODS);
} }
public List<Harvest> getDataDataset() throws Exception { public List<Harvest> getDataDataset() throws Exception {
String json = executeQuery("Dataset"); String json = executeQuery("Dataset");
return buildList(json, Harvest.NEW_CATALOGUE_DATASETS); return buildList(json, Harvest.NEW_CATALOGUE_DATASETS);
} }
public List<Harvest> getDataApplication() throws Exception { public List<Harvest> getDataApplication() throws Exception {
String json = executeQuery("Application"); String json = executeQuery("Application");
return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS); return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS);
} }
private List<Harvest> buildList(String json, int dataType) throws Exception { private List<Harvest> buildList(String json, int dataType) throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>(); ArrayList<Harvest> data = new ArrayList<Harvest>();
JSONObject jsonObject = new JSONObject(json); JSONObject jsonObject = new JSONObject(json);
JSONObject responseHeader = jsonObject.getJSONObject("responseHeader"); JSONObject responseHeader = jsonObject.getJSONObject("responseHeader");
int status = responseHeader.getInt("status"); int status = responseHeader.getInt("status");
if (status != 0) { if(status != 0) {
String err = "Query Deliverable in error: status " + status; String err = "Query Deliverable in error: status " + status;
logger.error(err); logger.error(err);
throw new Exception(err, null); throw new Exception(err, null);
} }
JSONObject response = jsonObject.getJSONObject("response"); JSONObject response = jsonObject.getJSONObject("response");
int numFound = response.getInt("numFound"); int numFound = response.getInt("numFound");
Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue", Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue",
numFound); numFound);
logger.debug(h.toString()); logger.debug(h.toString());
data.add(h); data.add(h);
if (numFound > 0) { if(numFound > 0) {
JSONArray docs = response.getJSONArray("docs"); JSONArray docs = response.getJSONArray("docs");
for (Object item : docs) { for(Object item : docs) {
JSONObject doc = (JSONObject) item; JSONObject doc = (JSONObject) item;
try { try {
JSONArray groups = doc.getJSONArray("groups"); JSONArray groups = doc.getJSONArray("groups");
Iterator<Object> git = groups.iterator(); Iterator<Object> git = groups.iterator();
while (git.hasNext()) { while(git.hasNext()) {
String groupItem = (String) git.next(); String groupItem = (String) git.next();
counterByGroup(groupItem); counterByGroup(groupItem);
} }
} catch (JSONException x) { } catch(JSONException x) {
logger.debug("Document without groups"); logger.debug("Document without groups");
} }
} }
h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens", h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens",
cityOfCitizensCounter); cityOfCitizensCounter);
logger.debug(h.toString()); logger.debug(h.toString());
@ -137,13 +126,13 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
wellBeingAndEconomyCounter); wellBeingAndEconomyCounter);
logger.debug(h.toString()); logger.debug(h.toString());
data.add(h); data.add(h);
} }
return data; return data;
} }
private String executeQuery(String fqSubString) throws Exception { private String executeQuery(String fqSubString) throws Exception {
String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?"; String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?";
String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO " String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO "
@ -152,49 +141,29 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\""); String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\"");
query += "&fq=" + fq + "&wt=json&indent=true"; query += "&fq=" + fq + "&wt=json&indent=true";
logger.debug(query); logger.debug(query);
String json = Utils.getJson(query); String json = Utils.getJson(query);
// logger.debug(json); // logger.debug(json);
return json; return json;
} }
private void counterByGroup(String groupName) { private void counterByGroup(String groupName) {
cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0; cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0;
switch (groupName) { switch(groupName) {
case "city-of-citizens-group": case "city-of-citizens-group":
cityOfCitizensCounter++; cityOfCitizensCounter++;
break; break;
case "migration-studies": case "migration-studies":
migrationStudiesCounter++; migrationStudiesCounter++;
break; break;
case "societal-debates-group": case "societal-debates-group":
societalDebatesCounter++; societalDebatesCounter++;
break; break;
case "well-being-and-economy-group": case "well-being-and-economy-group":
wellBeingAndEconomyCounter++; wellBeingAndEconomyCounter++;
break; break;
} }
} }
/*
* used only for test public static void main(String[] args) { Calendar from
* = Calendar.getInstance(); from.set(Calendar.DAY_OF_MONTH, 1);
* from.set(Calendar.MONTH, Calendar.NOVEMBER); from.set(Calendar.YEAR,
* 2017); from.set(Calendar.HOUR_OF_DAY, 0); from.set(Calendar.MINUTE, 0);
* from.set(Calendar.SECOND, 0);
*
* Calendar to = Calendar.getInstance(); to.set(Calendar.DAY_OF_MONTH, 30);
* to.set(Calendar.MONTH, Calendar.NOVEMBER); to.set(Calendar.YEAR, 2017);
* to.set(Calendar.HOUR_OF_DAY, 23); to.set(Calendar.MINUTE, 59);
* to.set(Calendar.SECOND, 59);
*
* try { ResourceCatalogueHarvester a = new
* ResourceCatalogueHarvester(from.getTime(), to.getTime()); List<Harvest> l
* = a.getData(); for(Harvest h : l) { System.out.println(h.toString()); }
* // System.out.println(Utils.dateToString(a.startDate)); //
* System.out.println(Utils.dateToStringWithTZ(a.endDate)); } catch
* (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); }
* }
*/
} }

51
src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java
View File

@ -16,32 +16,26 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class SocialHarvester extends BasicHarvester { public class SocialHarvester extends BasicHarvester {
private final String CATEGORY_NAME = "Accounting";
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class); private static Logger logger = LoggerFactory.getLogger(SocialHarvester.class);
private int likes, replies, posts; private int likes, replies, posts;
private long from = 0, to = 0; private long from = 0, to = 0;
public SocialHarvester(String start, String end) throws ParseException {
super(start, end);
this.from = startDate.getTime();
this.to = endDate.getTime();
}
public SocialHarvester(Date start, Date end) throws ParseException { public SocialHarvester(Date start, Date end) throws ParseException {
super(start, end); super(start, end);
this.from = startDate.getTime(); this.from = startDate.getTime();
this.to = endDate.getTime(); this.to = endDate.getTime();
} }
@Override @Override
public List<Harvest> getData() throws Exception { public List<Harvest> getData() throws Exception {
ArrayList<Harvest> data = new ArrayList<Harvest>(); ArrayList<Harvest> data = new ArrayList<Harvest>();
Properties props = readServiceEndpoint();
String[] vres = getActiveVREs(true); String[] vres = getActiveVREs(true);
for (String vre : vres) { for(String vre : vres) {
try { try {
logger.debug("Working on VRE: " + vre); logger.debug("Working on VRE: " + vre);
String key = vre; // vre.substring(vre.lastIndexOf("/") + 1); String key = vre; // vre.substring(vre.lastIndexOf("/") + 1);
String token = props.getProperty(key); String token = props.getProperty(key);
//System.out.println(key + ":" + token); //System.out.println(key + ":" + token);
@ -50,7 +44,7 @@ public class SocialHarvester extends BasicHarvester {
Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes); Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes);
data.add(harvest); data.add(harvest);
logger.debug(harvest.toString()); logger.debug(harvest.toString());
harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts); harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts);
data.add(harvest); data.add(harvest);
logger.debug(harvest.toString()); logger.debug(harvest.toString());
@ -58,52 +52,41 @@ public class SocialHarvester extends BasicHarvester {
harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies); harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies);
data.add(harvest); data.add(harvest);
logger.debug(harvest.toString()); logger.debug(harvest.toString());
} catch (Exception x) { } catch(Exception x) {
logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage()); logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage());
} }
} }
return data; return data;
} }
private void getJson(String token) throws MalformedURLException, IOException { private void getJson(String token) throws MalformedURLException, IOException {
likes = replies = posts = 0; likes = replies = posts = 0;
// la seguente stringa deve essere letta dinamicamente // la seguente stringa deve essere letta dinamicamente
String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token="; String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token=";
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
Boolean success = (Boolean) jsonObject.get("success"); Boolean success = (Boolean) jsonObject.get("success");
if (success == false) { if(success == false) {
String message = "getJson() on token: " + token + " success=false"; String message = "getJson() on token: " + token + " success=false";
logger.error(message); logger.error(message);
throw new IOException(message); throw new IOException(message);
} }
JSONArray res = jsonObject.getJSONArray("result"); JSONArray res = jsonObject.getJSONArray("result");
int len = res.length(); int len = res.length();
for (int i = 0; i < len; i++) { for(int i = 0; i < len; i++) {
JSONObject item = res.getJSONObject(i); JSONObject item = res.getJSONObject(i);
long time = item.getLong("time"); long time = item.getLong("time");
//System.out.println(from + " - " + time + " - " + to ); //System.out.println(from + " - " + time + " - " + to );
if ((from <= time) && (time <= to)) { if((from <= time) && (time <= to)) {
posts++; posts++;
replies += item.getInt("comments_no"); replies += item.getInt("comments_no");
likes += item.getInt("likes_no"); likes += item.getInt("likes_no");
} }
} }
} }
/**/
public static void main(String[] argv) {
try {
SocialHarvester a = new SocialHarvester("2018-01-01 00:00:00", "2018-01-31 23:59:59");
List<Harvest> list = a.getData();
for (Harvest l : list) {
System.out.println(l.toString());
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}/**/
} }

67
src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java
View File

@ -15,13 +15,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class VreUsersHarvester extends BasicHarvester { public class VreUsersHarvester extends BasicHarvester {
// private final String CATEGORY_NAME = "Accounting";
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class); private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
public VreUsersHarvester(String start, String end) throws ParseException {
super(start, end);
}
public VreUsersHarvester(Date start, Date end) throws ParseException { public VreUsersHarvester(Date start, Date end) throws ParseException {
super(start, end); super(start, end);
} }
@ -60,63 +57,5 @@ public class VreUsersHarvester extends BasicHarvester {
userNumber = jsonObject.getJSONArray("result").length(); userNumber = jsonObject.getJSONArray("result").length();
return userNumber; return userNumber;
} }
// private Properties readServiceEndpoint() throws Exception {
// Properties props = new Properties();
// String scope = "/d4science.research-infrastructures.eu";
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
//
// SimpleQuery query = queryFor(ServiceEndpoint.class);
// query.addCondition("$resource/Profile/Name/text() eq '" +
// RUNTIME_RESOURCE_NAME + "'");
// query.addCondition("$resource/Profile/Category/text() eq '" +
// CATEGORY_NAME + "'");
//
// DiscoveryClient<ServiceEndpoint> client =
// clientFor(ServiceEndpoint.class);
// String password = null;
// try {
// List<ServiceEndpoint> list = client.submit(query);
// if (list.size() > 1) {
// logger.error("Too many Service Endpoints having name " +
// RUNTIME_RESOURCE_NAME
// + " in this scope having Category " + CATEGORY_NAME);
// } else if (list.size() == 0) {
// logger.error("There is no Service Endpoint having name " +
// RUNTIME_RESOURCE_NAME
// + " and Category " + CATEGORY_NAME + " in this scope " + scope);
// } else {
// for (ServiceEndpoint res : list) {
// AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
// .toArray(new AccessPoint[res.profile().accessPoints().size()]);
// for (AccessPoint found : accessPoints) {
// password =
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
// for (ServiceEndpoint.Property prop : found.properties()) {
// props.setProperty(prop.name(),
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
// }
// }
//
// }
// }
// return props;
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage());
// throw e;
// }
// }
// public static void main(String[] argv) {
// try {
// VreUsersHarvester a = new VreUsersHarvester("2017-01-01", "2017-02-01");
// List<Harvest> list = a.getData();
// for(Harvest l : list) {
// System.out.println(l.toString());
// }
// } catch (Exception e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// }
} }

7
src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java
View File

@ -9,6 +9,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.TreeMap;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.authorization.library.provider.UserInfo; import org.gcube.common.authorization.library.provider.UserInfo;
@ -41,12 +42,14 @@ public class ContextAuthorization {
*/ */
protected Map<String, String> tokenToContext; protected Map<String, String> tokenToContext;
/** /**
* Contains Properties used to generate tokens * Contains Properties used to generate tokens
*/ */
public ContextAuthorization() throws Exception { public ContextAuthorization() throws Exception {
this.contextToToken = new HashMap<>(); this.contextToToken = new TreeMap<>();
this.tokenToContext = new HashMap<>(); this.tokenToContext = new HashMap<>();
retrieveContextsAndTokens(); retrieveContextsAndTokens();
} }

4
src/main/java/org/gcube/dataharvest/utils/Utils.java
View File

@ -25,10 +25,6 @@ public class Utils {
return cal; return cal;
} }
public static String dateToString(Date date) {
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
return formatter.format(date);
}
public static String dateToStringWithTZ(Date date) { public static String dateToStringWithTZ(Date date) {
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");