Code redesign
Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167568 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
9dec451f4c
commit
8d64022afa
|
|
@ -72,28 +72,11 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
|
||||
};
|
||||
|
||||
public static InheritableThreadLocal<ContextAuthorization> getContextAuthorization() {
|
||||
return contextAuthorization;
|
||||
}
|
||||
|
||||
private void retrieveAuthorizations() throws Exception {
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
getContextAuthorization().set(contextAuthorization);
|
||||
}
|
||||
|
||||
public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) {
|
||||
super(pluginDeclaration);
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
@Override
|
||||
public void launch(Map<String,Object> inputs) throws Exception {
|
||||
logger.debug("{} is starting", this.getClass().getSimpleName());
|
||||
|
||||
getConfigParameters();
|
||||
|
||||
retrieveAuthorizations();
|
||||
|
||||
if(inputs == null || inputs.isEmpty()) {
|
||||
throw new IllegalArgumentException("The can only be launched providing valid input parameters");
|
||||
}
|
||||
|
|
@ -134,63 +117,71 @@ public class AccountingDataHarvesterPlugin extends Plugin<DataHarvestPluginDecla
|
|||
logger.debug("Harvesting from {} to {}", DateUtils.LAUNCH_DATE_FORMAT.format(start),
|
||||
DateUtils.LAUNCH_DATE_FORMAT.format(end));
|
||||
|
||||
getConfigParameters();
|
||||
|
||||
ContextAuthorization contextAuthorization = new ContextAuthorization();
|
||||
|
||||
DatabaseManager dbaseManager = new DatabaseManager();
|
||||
|
||||
try {
|
||||
// collecting info on VRE users
|
||||
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
|
||||
List<Harvest> users = vreUsersHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, users, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
|
||||
List<Harvest> res = resourceCatalogueHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
for() {
|
||||
|
||||
try {
|
||||
// collecting info on Data/Method download
|
||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
|
||||
List<Harvest> res = dataMethodDownloadHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
// collecting info on social (posts, replies and likes)
|
||||
SocialHarvester socialHarvester = new SocialHarvester(start, end);
|
||||
List<Harvest> res = socialHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
try {
|
||||
// collecting info on VRE users
|
||||
VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end);
|
||||
List<Harvest> users = vreUsersHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, users, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
// collecting info on method invocation
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<Harvest> res = methodInvocationHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
try {
|
||||
// collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods)
|
||||
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end);
|
||||
List<Harvest> res = resourceCatalogueHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
// collecting info on Data/Method download
|
||||
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end);
|
||||
List<Harvest> res = dataMethodDownloadHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
// collecting info on social (posts, replies and likes)
|
||||
SocialHarvester socialHarvester = new SocialHarvester(start, end);
|
||||
List<Harvest> res = socialHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
// collecting info on method invocation
|
||||
MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end);
|
||||
List<Harvest> res = methodInvocationHarvester.getData();
|
||||
if(!dryRun) {
|
||||
dbaseManager.insertMonthlyData(start, end, res, reRun);
|
||||
}
|
||||
} catch(Exception x) {
|
||||
logger.error(x.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** {@inheritDoc} */
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@ import java.util.Date;
|
|||
import java.io.Serializable;
|
||||
|
||||
public class Harvest implements Serializable {
|
||||
|
||||
/**
|
||||
*
|
||||
* Generated Serial Version UID
|
||||
*/
|
||||
private static final long serialVersionUID = 1L;
|
||||
private static final long serialVersionUID = 3699669951917080213L;
|
||||
|
||||
public static int ACCESSESS = 1;
|
||||
public static int USERS = 2;
|
||||
public static int DATA_METHOD_DOWNLOAD = 3;
|
||||
|
|
|
|||
|
|
@ -1,47 +1,28 @@
|
|||
package org.gcube.dataharvest.harvester;
|
||||
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.gcube.common.authorization.client.Constants;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint.Property;
|
||||
import org.gcube.common.resources.gcore.utils.Group;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.dataharvest.dao.DatabaseManager;
|
||||
import org.gcube.dataharvest.datamodel.Harvest;
|
||||
import org.gcube.informationsystem.publisher.RegistryPublisher;
|
||||
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class BasicHarvester {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class);
|
||||
public Date startDate = null, endDate = null;
|
||||
public DateFormat format;
|
||||
|
||||
public Date startDate;
|
||||
public Date endDate;
|
||||
|
||||
public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester";
|
||||
public final String CATEGORY_NAME = "Accounting";
|
||||
|
||||
public BasicHarvester(String start, String end) throws ParseException {
|
||||
format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
|
||||
startDate = format.parse(start);
|
||||
endDate = format.parse(end);
|
||||
}
|
||||
|
||||
public BasicHarvester(Date start, Date end) throws ParseException {
|
||||
startDate = start;
|
||||
endDate = end;
|
||||
|
|
@ -69,86 +50,6 @@ public class BasicHarvester {
|
|||
return null;
|
||||
}
|
||||
|
||||
public Properties readServiceEndpoint() throws Exception {
|
||||
Properties props = new Properties();
|
||||
String scope = "/d4science.research-infrastructures.eu";
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||
|
||||
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
|
||||
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
|
||||
|
||||
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||
String password = null;
|
||||
try {
|
||||
List<ServiceEndpoint> list = client.submit(query);
|
||||
if (list.size() > 1) {
|
||||
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
|
||||
+ " in this scope having Category " + CATEGORY_NAME);
|
||||
} else if (list.size() == 0) {
|
||||
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
|
||||
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
|
||||
} else {
|
||||
for (ServiceEndpoint res : list) {
|
||||
AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
|
||||
.toArray(new AccessPoint[res.profile().accessPoints().size()]);
|
||||
for (AccessPoint found : accessPoints) {
|
||||
password = org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
|
||||
for (ServiceEndpoint.Property prop : found.properties()) {
|
||||
props.setProperty(prop.name(), org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return props;
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getLocalizedMessage());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public void updateServiceEndPoint(ArrayList<Property> newProps) throws Exception {
|
||||
String scope = "/d4science.research-infrastructures.eu";
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||
|
||||
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'");
|
||||
query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'");
|
||||
|
||||
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||
String password = null;
|
||||
try {
|
||||
List<ServiceEndpoint> list = client.submit(query);
|
||||
if (list.size() > 1) {
|
||||
logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME
|
||||
+ " in this scope having Category " + CATEGORY_NAME);
|
||||
} else if (list.size() == 0) {
|
||||
logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME
|
||||
+ " and Category " + CATEGORY_NAME + " in this scope " + scope);
|
||||
} else {
|
||||
ServiceEndpoint serviceEndpoint = list.get(0);
|
||||
AccessPoint accessPoint = ((AccessPoint[]) serviceEndpoint.profile().accessPoints().toArray(new AccessPoint[serviceEndpoint.profile().accessPoints().size()]))[0];
|
||||
Group<Property> oldProps = accessPoint.properties();
|
||||
for (Property prop : newProps) {
|
||||
String propValue = prop.value();
|
||||
propValue = org.gcube.common.encryption.StringEncrypter.getEncrypter().encrypt(propValue);
|
||||
String propKey = prop.name();
|
||||
org.gcube.common.resources.gcore.ServiceEndpoint.Property pToAdd =
|
||||
new org.gcube.common.resources.gcore.ServiceEndpoint.Property().nameAndValue(propKey, propValue);
|
||||
pToAdd.encrypted(true);
|
||||
accessPoint.properties().add(pToAdd);
|
||||
}
|
||||
|
||||
RegistryPublisher publisher = RegistryPublisherFactory.create();
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||
publisher.update(serviceEndpoint);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getLocalizedMessage());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public String[] getActiveVREs(boolean testMode) {
|
||||
DatabaseManager dbaseManager = new DatabaseManager();
|
||||
return dbaseManager.getActiveVres();
|
||||
|
|
|
|||
|
|
@ -1,81 +1,58 @@
|
|||
package org.gcube.dataharvest.harvester;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.dataharvest.AccountingDataHarvesterPlugin;
|
||||
import org.gcube.dataharvest.datamodel.Harvest;
|
||||
import org.gcube.dataharvest.utils.Utils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.commons.lang.Validate;
|
||||
import org.gcube.common.homelibrary.home.Home;
|
||||
import org.gcube.common.homelibrary.home.HomeLibrary;
|
||||
import org.gcube.common.homelibrary.home.HomeManager;
|
||||
import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException;
|
||||
import org.gcube.common.homelibrary.home.exceptions.InternalErrorException;
|
||||
import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException;
|
||||
import org.gcube.common.homelibrary.home.workspace.WorkspaceItem;
|
||||
import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry;
|
||||
import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException;
|
||||
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException;
|
||||
import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException;
|
||||
import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException;
|
||||
import org.gcube.common.homelibrary.home.workspace.exceptions.WrongDestinationException;
|
||||
import org.gcube.common.homelibrary.jcr.repository.JCRRepository;
|
||||
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace;
|
||||
import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem;
|
||||
import org.gcube.common.homelibrary.model.exceptions.RepositoryException;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.dataharvest.datamodel.Harvest;
|
||||
import org.gcube.dataharvest.utils.Utils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class DataMethodDownloadHarvester extends BasicHarvester {
|
||||
private Date startDate, endDate;
|
||||
private DateFormat format;
|
||||
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
|
||||
private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class);
|
||||
private int count = 0;
|
||||
|
||||
public DataMethodDownloadHarvester(String start, String end) throws ParseException {
|
||||
super(start, end);
|
||||
//ScopeProvider.instance.set(SCOPE);
|
||||
}
|
||||
private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class);
|
||||
|
||||
private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu";
|
||||
|
||||
private int count = 0;
|
||||
|
||||
public DataMethodDownloadHarvester(Date start, Date end) throws ParseException {
|
||||
super(start, end);
|
||||
//ScopeProvider.instance.set(SCOPE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Harvest> getData() throws Exception {
|
||||
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||
|
||||
ScopeProvider.instance.set(SCOPE);
|
||||
|
||||
String[] vres = getActiveVREs(true);
|
||||
for (String vre : vres) {
|
||||
try {
|
||||
count = 0;
|
||||
|
||||
logger.error("HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager()");
|
||||
|
||||
HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager();
|
||||
|
||||
String user = getVREName(vre) + "-Manager";
|
||||
|
||||
logger.error("Home home = manager.getHome(user)");
|
||||
|
||||
Home home = manager.getHome(user);
|
||||
|
||||
logger.error("JCRWorkspace ws = (JCRWorkspace) home.getWorkspace()");
|
||||
|
||||
JCRWorkspace ws = (JCRWorkspace) home.getWorkspace();
|
||||
|
||||
logger.error("ws.getItemByPath...");
|
||||
|
||||
JCRWorkspaceItem item = (JCRWorkspaceItem) ws
|
||||
.getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre));
|
||||
|
||||
|
|
|
|||
|
|
@ -3,25 +3,20 @@ package org.gcube.dataharvest.harvester;
|
|||
import java.text.ParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.analytics.Filter;
|
||||
import org.gcube.accounting.analytics.Info;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint;
|
||||
import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery;
|
||||
import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory;
|
||||
import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord;
|
||||
import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord;
|
||||
|
||||
import java.util.SortedMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.gcube.dataharvest.datamodel.Harvest;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -30,10 +25,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
|
|||
|
||||
private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class);
|
||||
|
||||
public MethodInvocationHarvester(String start, String end) throws ParseException {
|
||||
super(start, end);
|
||||
}
|
||||
|
||||
public MethodInvocationHarvester(Date start, Date end) throws ParseException {
|
||||
super(start, end);
|
||||
}
|
||||
|
|
@ -83,33 +74,6 @@ public class MethodInvocationHarvester extends BasicHarvester {
|
|||
logger.error(errorMessage);
|
||||
throw x;
|
||||
}
|
||||
}/**/
|
||||
|
||||
/* used only for test */
|
||||
public static void main(String[] args) {
|
||||
String ROOT = "18fed2d9-030b-4c77-93af-af2015d945f7-843339462";
|
||||
String GCUBE = "32542bb3-b04b-4112-9af1-7fac049d33cd-98187548";
|
||||
String TAGME = "308cf6bd-3cbb-4267-a300-0cf5ae122bc9-843339462";
|
||||
String TAGME2 = "9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462";
|
||||
String SOBIGDATA_EU = "6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462";
|
||||
try {
|
||||
MethodInvocationHarvester m = new MethodInvocationHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59");
|
||||
MethodInvocationHarvester.setContext(SOBIGDATA_EU);
|
||||
System.out.println(MethodInvocationHarvester.getCurrentContext());
|
||||
Properties props = m.readServiceEndpoint();
|
||||
System.out.println("Found " + props.size() + " properties.");
|
||||
Set keys = props.keySet();
|
||||
for(Object key : keys) {
|
||||
System.out.println((String)key + " :: " + props.getProperty((String) key));
|
||||
}
|
||||
|
||||
m.getData();
|
||||
} catch (Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
System.out.println(e.getLocalizedMessage());
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
/**/
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,28 @@
|
|||
package org.gcube.dataharvest.harvester;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.http.client.utils.URLEncodedUtils;
|
||||
import org.gcube.dataharvest.datamodel.Harvest;
|
||||
import org.gcube.dataharvest.utils.Utils;
|
||||
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.quartz.utils.StringKeyDirtyFlagMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ResourceCatalogueHarvester extends BasicHarvester {
|
||||
private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class);
|
||||
private int cityOfCitizensCounter = 0, migrationStudiesCounter = 0, societalDebatesCounter = 0,
|
||||
wellBeingAndEconomyCounter = 0;
|
||||
|
||||
public ResourceCatalogueHarvester(String start, String end) throws ParseException {
|
||||
super(start, end);
|
||||
}
|
||||
private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class);
|
||||
|
||||
private int cityOfCitizensCounter = 0;
|
||||
private int migrationStudiesCounter = 0;
|
||||
private int societalDebatesCounter = 0;
|
||||
private int wellBeingAndEconomyCounter = 0;
|
||||
|
||||
public ResourceCatalogueHarvester(Date start, Date end) throws ParseException {
|
||||
super(start, end);
|
||||
|
|
@ -42,24 +30,25 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
|
|||
|
||||
@Override
|
||||
public List<Harvest> getData() throws Exception {
|
||||
|
||||
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||
List<Harvest> dataDeliverable = getDataDeliverable();
|
||||
for (Harvest harvest : dataDeliverable) {
|
||||
for(Harvest harvest : dataDeliverable) {
|
||||
data.add(harvest);
|
||||
}
|
||||
|
||||
List<Harvest> dataMethod = getDataMethod();
|
||||
for (Harvest harvest : dataMethod) {
|
||||
for(Harvest harvest : dataMethod) {
|
||||
data.add(harvest);
|
||||
}
|
||||
|
||||
List<Harvest> dataDataset = getDataDataset();
|
||||
for (Harvest harvest : dataDataset) {
|
||||
for(Harvest harvest : dataDataset) {
|
||||
data.add(harvest);
|
||||
}
|
||||
|
||||
List<Harvest> dataApplication = getDataApplication();
|
||||
for (Harvest harvest : dataApplication) {
|
||||
for(Harvest harvest : dataApplication) {
|
||||
data.add(harvest);
|
||||
}
|
||||
|
||||
|
|
@ -92,7 +81,7 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
|
|||
|
||||
JSONObject responseHeader = jsonObject.getJSONObject("responseHeader");
|
||||
int status = responseHeader.getInt("status");
|
||||
if (status != 0) {
|
||||
if(status != 0) {
|
||||
String err = "Query Deliverable in error: status " + status;
|
||||
logger.error(err);
|
||||
throw new Exception(err, null);
|
||||
|
|
@ -105,19 +94,19 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
|
|||
numFound);
|
||||
logger.debug(h.toString());
|
||||
data.add(h);
|
||||
if (numFound > 0) {
|
||||
if(numFound > 0) {
|
||||
|
||||
JSONArray docs = response.getJSONArray("docs");
|
||||
for (Object item : docs) {
|
||||
for(Object item : docs) {
|
||||
JSONObject doc = (JSONObject) item;
|
||||
try {
|
||||
JSONArray groups = doc.getJSONArray("groups");
|
||||
Iterator<Object> git = groups.iterator();
|
||||
while (git.hasNext()) {
|
||||
while(git.hasNext()) {
|
||||
String groupItem = (String) git.next();
|
||||
counterByGroup(groupItem);
|
||||
}
|
||||
} catch (JSONException x) {
|
||||
} catch(JSONException x) {
|
||||
logger.debug("Document without groups");
|
||||
}
|
||||
}
|
||||
|
|
@ -161,40 +150,20 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
|
|||
|
||||
private void counterByGroup(String groupName) {
|
||||
cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0;
|
||||
switch (groupName) {
|
||||
case "city-of-citizens-group":
|
||||
cityOfCitizensCounter++;
|
||||
break;
|
||||
case "migration-studies":
|
||||
migrationStudiesCounter++;
|
||||
break;
|
||||
case "societal-debates-group":
|
||||
societalDebatesCounter++;
|
||||
break;
|
||||
case "well-being-and-economy-group":
|
||||
wellBeingAndEconomyCounter++;
|
||||
break;
|
||||
switch(groupName) {
|
||||
case "city-of-citizens-group":
|
||||
cityOfCitizensCounter++;
|
||||
break;
|
||||
case "migration-studies":
|
||||
migrationStudiesCounter++;
|
||||
break;
|
||||
case "societal-debates-group":
|
||||
societalDebatesCounter++;
|
||||
break;
|
||||
case "well-being-and-economy-group":
|
||||
wellBeingAndEconomyCounter++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* used only for test public static void main(String[] args) { Calendar from
|
||||
* = Calendar.getInstance(); from.set(Calendar.DAY_OF_MONTH, 1);
|
||||
* from.set(Calendar.MONTH, Calendar.NOVEMBER); from.set(Calendar.YEAR,
|
||||
* 2017); from.set(Calendar.HOUR_OF_DAY, 0); from.set(Calendar.MINUTE, 0);
|
||||
* from.set(Calendar.SECOND, 0);
|
||||
*
|
||||
* Calendar to = Calendar.getInstance(); to.set(Calendar.DAY_OF_MONTH, 30);
|
||||
* to.set(Calendar.MONTH, Calendar.NOVEMBER); to.set(Calendar.YEAR, 2017);
|
||||
* to.set(Calendar.HOUR_OF_DAY, 23); to.set(Calendar.MINUTE, 59);
|
||||
* to.set(Calendar.SECOND, 59);
|
||||
*
|
||||
* try { ResourceCatalogueHarvester a = new
|
||||
* ResourceCatalogueHarvester(from.getTime(), to.getTime()); List<Harvest> l
|
||||
* = a.getData(); for(Harvest h : l) { System.out.println(h.toString()); }
|
||||
* // System.out.println(Utils.dateToString(a.startDate)); //
|
||||
* System.out.println(Utils.dateToStringWithTZ(a.endDate)); } catch
|
||||
* (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); }
|
||||
* }
|
||||
*/
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,17 +16,12 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class SocialHarvester extends BasicHarvester {
|
||||
private final String CATEGORY_NAME = "Accounting";
|
||||
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(SocialHarvester.class);
|
||||
|
||||
private int likes, replies, posts;
|
||||
private long from = 0, to = 0;
|
||||
|
||||
public SocialHarvester(String start, String end) throws ParseException {
|
||||
super(start, end);
|
||||
this.from = startDate.getTime();
|
||||
this.to = endDate.getTime();
|
||||
}
|
||||
|
||||
public SocialHarvester(Date start, Date end) throws ParseException {
|
||||
super(start, end);
|
||||
this.from = startDate.getTime();
|
||||
|
|
@ -36,9 +31,8 @@ public class SocialHarvester extends BasicHarvester {
|
|||
@Override
|
||||
public List<Harvest> getData() throws Exception {
|
||||
ArrayList<Harvest> data = new ArrayList<Harvest>();
|
||||
Properties props = readServiceEndpoint();
|
||||
String[] vres = getActiveVREs(true);
|
||||
for (String vre : vres) {
|
||||
for(String vre : vres) {
|
||||
try {
|
||||
logger.debug("Working on VRE: " + vre);
|
||||
|
||||
|
|
@ -58,7 +52,7 @@ public class SocialHarvester extends BasicHarvester {
|
|||
harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies);
|
||||
data.add(harvest);
|
||||
logger.debug(harvest.toString());
|
||||
} catch (Exception x) {
|
||||
} catch(Exception x) {
|
||||
logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +66,7 @@ public class SocialHarvester extends BasicHarvester {
|
|||
JSONObject jsonObject = new JSONObject(Utils.getJson(url + token));
|
||||
|
||||
Boolean success = (Boolean) jsonObject.get("success");
|
||||
if (success == false) {
|
||||
if(success == false) {
|
||||
String message = "getJson() on token: " + token + " success=false";
|
||||
logger.error(message);
|
||||
throw new IOException(message);
|
||||
|
|
@ -81,11 +75,11 @@ public class SocialHarvester extends BasicHarvester {
|
|||
JSONArray res = jsonObject.getJSONArray("result");
|
||||
int len = res.length();
|
||||
|
||||
for (int i = 0; i < len; i++) {
|
||||
for(int i = 0; i < len; i++) {
|
||||
JSONObject item = res.getJSONObject(i);
|
||||
long time = item.getLong("time");
|
||||
//System.out.println(from + " - " + time + " - " + to );
|
||||
if ((from <= time) && (time <= to)) {
|
||||
if((from <= time) && (time <= to)) {
|
||||
posts++;
|
||||
replies += item.getInt("comments_no");
|
||||
likes += item.getInt("likes_no");
|
||||
|
|
@ -93,17 +87,6 @@ public class SocialHarvester extends BasicHarvester {
|
|||
}
|
||||
}
|
||||
|
||||
/**/
|
||||
public static void main(String[] argv) {
|
||||
try {
|
||||
SocialHarvester a = new SocialHarvester("2018-01-01 00:00:00", "2018-01-31 23:59:59");
|
||||
List<Harvest> list = a.getData();
|
||||
for (Harvest l : list) {
|
||||
System.out.println(l.toString());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}/**/
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,12 +15,9 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class VreUsersHarvester extends BasicHarvester {
|
||||
// private final String CATEGORY_NAME = "Accounting";
|
||||
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
|
||||
|
||||
public VreUsersHarvester(String start, String end) throws ParseException {
|
||||
super(start, end);
|
||||
}
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class);
|
||||
|
||||
public VreUsersHarvester(Date start, Date end) throws ParseException {
|
||||
super(start, end);
|
||||
|
|
@ -61,62 +58,4 @@ public class VreUsersHarvester extends BasicHarvester {
|
|||
return userNumber;
|
||||
}
|
||||
|
||||
// private Properties readServiceEndpoint() throws Exception {
|
||||
// Properties props = new Properties();
|
||||
// String scope = "/d4science.research-infrastructures.eu";
|
||||
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu");
|
||||
//
|
||||
// SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
// query.addCondition("$resource/Profile/Name/text() eq '" +
|
||||
// RUNTIME_RESOURCE_NAME + "'");
|
||||
// query.addCondition("$resource/Profile/Category/text() eq '" +
|
||||
// CATEGORY_NAME + "'");
|
||||
//
|
||||
// DiscoveryClient<ServiceEndpoint> client =
|
||||
// clientFor(ServiceEndpoint.class);
|
||||
// String password = null;
|
||||
// try {
|
||||
// List<ServiceEndpoint> list = client.submit(query);
|
||||
// if (list.size() > 1) {
|
||||
// logger.error("Too many Service Endpoints having name " +
|
||||
// RUNTIME_RESOURCE_NAME
|
||||
// + " in this scope having Category " + CATEGORY_NAME);
|
||||
// } else if (list.size() == 0) {
|
||||
// logger.error("There is no Service Endpoint having name " +
|
||||
// RUNTIME_RESOURCE_NAME
|
||||
// + " and Category " + CATEGORY_NAME + " in this scope " + scope);
|
||||
// } else {
|
||||
// for (ServiceEndpoint res : list) {
|
||||
// AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints()
|
||||
// .toArray(new AccessPoint[res.profile().accessPoints().size()]);
|
||||
// for (AccessPoint found : accessPoints) {
|
||||
// password =
|
||||
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password());
|
||||
// for (ServiceEndpoint.Property prop : found.properties()) {
|
||||
// props.setProperty(prop.name(),
|
||||
// org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value()));
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// return props;
|
||||
// } catch (Exception e) {
|
||||
// logger.error(e.getLocalizedMessage());
|
||||
// throw e;
|
||||
// }
|
||||
// }
|
||||
|
||||
// public static void main(String[] argv) {
|
||||
// try {
|
||||
// VreUsersHarvester a = new VreUsersHarvester("2017-01-01", "2017-02-01");
|
||||
// List<Harvest> list = a.getData();
|
||||
// for(Harvest l : list) {
|
||||
// System.out.println(l.toString());
|
||||
// }
|
||||
// } catch (Exception e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import java.util.HashMap;
|
|||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.authorization.library.provider.UserInfo;
|
||||
|
|
@ -41,12 +42,14 @@ public class ContextAuthorization {
|
|||
*/
|
||||
protected Map<String, String> tokenToContext;
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Contains Properties used to generate tokens
|
||||
*/
|
||||
|
||||
public ContextAuthorization() throws Exception {
|
||||
this.contextToToken = new HashMap<>();
|
||||
this.contextToToken = new TreeMap<>();
|
||||
this.tokenToContext = new HashMap<>();
|
||||
retrieveContextsAndTokens();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,10 +25,6 @@ public class Utils {
|
|||
return cal;
|
||||
}
|
||||
|
||||
public static String dateToString(Date date) {
|
||||
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
|
||||
return formatter.format(date);
|
||||
}
|
||||
|
||||
public static String dateToStringWithTZ(Date date) {
|
||||
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
||||
|
|
|
|||
Loading…
Reference in New Issue