diff --git a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java index b46b952..21cc835 100644 --- a/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java +++ b/src/main/java/org/gcube/dataharvest/AccountingDataHarvesterPlugin.java @@ -72,28 +72,11 @@ public class AccountingDataHarvesterPlugin extends Plugin getContextAuthorization() { - return contextAuthorization; - } - - private void retrieveAuthorizations() throws Exception { - ContextAuthorization contextAuthorization = new ContextAuthorization(); - getContextAuthorization().set(contextAuthorization); - } - - public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) { - super(pluginDeclaration); - } - /** {@inheritDoc} */ @Override public void launch(Map inputs) throws Exception { logger.debug("{} is starting", this.getClass().getSimpleName()); - getConfigParameters(); - - retrieveAuthorizations(); - if(inputs == null || inputs.isEmpty()) { throw new IllegalArgumentException("The can only be launched providing valid input parameters"); } @@ -134,63 +117,71 @@ public class AccountingDataHarvesterPlugin extends Plugin users = vreUsersHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, users, reRun); - } - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - try { - // collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods) - ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); - List res = resourceCatalogueHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); - } - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } + for() { - try { - // collecting info on Data/Method download - DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end); - List res = dataMethodDownloadHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); + + try { + // collecting info on VRE users + VreUsersHarvester vreUsersHarvester = new VreUsersHarvester(start, end); + List users = vreUsersHarvester.getData(); + if(!dryRun) { + dbaseManager.insertMonthlyData(start, end, users, reRun); + } + } catch(Exception x) { + logger.error(x.getLocalizedMessage()); } - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - - try { - // collecting info on social (posts, replies and likes) - SocialHarvester socialHarvester = new SocialHarvester(start, end); - List res = socialHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); + + try { + // collecting info on Res. Catalogue (Dataset, Application, Deliverables, Methods) + ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end); + List res = resourceCatalogueHarvester.getData(); + if(!dryRun) { + dbaseManager.insertMonthlyData(start, end, res, reRun); + } + } catch(Exception x) { + logger.error(x.getLocalizedMessage()); } - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); - } - - try { - // collecting info on method invocation - MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); - List res = methodInvocationHarvester.getData(); - if(!dryRun) { - dbaseManager.insertMonthlyData(start, end, res, reRun); + + try { + // collecting info on Data/Method download + DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end); + List res = dataMethodDownloadHarvester.getData(); + if(!dryRun) { + dbaseManager.insertMonthlyData(start, end, res, reRun); + } + } catch(Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on social (posts, replies and likes) + SocialHarvester socialHarvester = new SocialHarvester(start, end); + List res = socialHarvester.getData(); + if(!dryRun) { + dbaseManager.insertMonthlyData(start, end, res, reRun); + } + } catch(Exception x) { + logger.error(x.getLocalizedMessage()); + } + + try { + // collecting info on method invocation + MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); + List res = methodInvocationHarvester.getData(); + if(!dryRun) { + dbaseManager.insertMonthlyData(start, end, res, reRun); + } + } catch(Exception x) { + logger.error(x.getLocalizedMessage()); } - } catch(Exception x) { - logger.error(x.getLocalizedMessage()); } - } /** {@inheritDoc} */ diff --git a/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java b/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java index a8b982a..b05e406 100644 --- a/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java +++ b/src/main/java/org/gcube/dataharvest/datamodel/Harvest.java @@ -4,10 +4,12 @@ import java.util.Date; import java.io.Serializable; public class Harvest implements Serializable { + /** - * + * Generated Serial Version UID */ - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 3699669951917080213L; + public static int ACCESSESS = 1; public static int USERS = 2; public static int DATA_METHOD_DOWNLOAD = 3; diff --git a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java index 9d3d7fc..3e09bcc 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/BasicHarvester.java @@ -1,47 +1,28 @@ package org.gcube.dataharvest.harvester; -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - -import java.text.DateFormat; import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Locale; -import java.util.Properties; import org.gcube.common.authorization.client.Constants; import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.resources.gcore.ServiceEndpoint; -import org.gcube.common.resources.gcore.ServiceEndpoint.AccessPoint; -import org.gcube.common.resources.gcore.ServiceEndpoint.Property; -import org.gcube.common.resources.gcore.utils.Group; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.Harvest; -import org.gcube.informationsystem.publisher.RegistryPublisher; -import org.gcube.informationsystem.publisher.RegistryPublisherFactory; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BasicHarvester { + private static Logger logger = LoggerFactory.getLogger(BasicHarvester.class); - public Date startDate = null, endDate = null; - public DateFormat format; + + public Date startDate; + public Date endDate; + public final String RUNTIME_RESOURCE_NAME = "AccountingDataHarvester"; public final String CATEGORY_NAME = "Accounting"; - public BasicHarvester(String start, String end) throws ParseException { - format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH); - startDate = format.parse(start); - endDate = format.parse(end); - } - public BasicHarvester(Date start, Date end) throws ParseException { startDate = start; endDate = end; @@ -69,86 +50,6 @@ public class BasicHarvester { return null; } - public Properties readServiceEndpoint() throws Exception { - Properties props = new Properties(); - String scope = "/d4science.research-infrastructures.eu"; - ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); - - SimpleQuery query = queryFor(ServiceEndpoint.class); - query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'"); - query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'"); - - DiscoveryClient client = clientFor(ServiceEndpoint.class); - String password = null; - try { - List list = client.submit(query); - if (list.size() > 1) { - logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME - + " in this scope having Category " + CATEGORY_NAME); - } else if (list.size() == 0) { - logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME - + " and Category " + CATEGORY_NAME + " in this scope " + scope); - } else { - for (ServiceEndpoint res : list) { - AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints() - .toArray(new AccessPoint[res.profile().accessPoints().size()]); - for (AccessPoint found : accessPoints) { - password = org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password()); - for (ServiceEndpoint.Property prop : found.properties()) { - props.setProperty(prop.name(), org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value())); - } - } - } - } - return props; - } catch (Exception e) { - logger.error(e.getLocalizedMessage()); - throw e; - } - } - - public void updateServiceEndPoint(ArrayList newProps) throws Exception { - String scope = "/d4science.research-infrastructures.eu"; - ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); - - SimpleQuery query = queryFor(ServiceEndpoint.class); - query.addCondition("$resource/Profile/Name/text() eq '" + RUNTIME_RESOURCE_NAME + "'"); - query.addCondition("$resource/Profile/Category/text() eq '" + CATEGORY_NAME + "'"); - - DiscoveryClient client = clientFor(ServiceEndpoint.class); - String password = null; - try { - List list = client.submit(query); - if (list.size() > 1) { - logger.error("Too many Service Endpoints having name " + RUNTIME_RESOURCE_NAME - + " in this scope having Category " + CATEGORY_NAME); - } else if (list.size() == 0) { - logger.error("There is no Service Endpoint having name " + RUNTIME_RESOURCE_NAME - + " and Category " + CATEGORY_NAME + " in this scope " + scope); - } else { - ServiceEndpoint serviceEndpoint = list.get(0); - AccessPoint accessPoint = ((AccessPoint[]) serviceEndpoint.profile().accessPoints().toArray(new AccessPoint[serviceEndpoint.profile().accessPoints().size()]))[0]; - Group oldProps = accessPoint.properties(); - for (Property prop : newProps) { - String propValue = prop.value(); - propValue = org.gcube.common.encryption.StringEncrypter.getEncrypter().encrypt(propValue); - String propKey = prop.name(); - org.gcube.common.resources.gcore.ServiceEndpoint.Property pToAdd = - new org.gcube.common.resources.gcore.ServiceEndpoint.Property().nameAndValue(propKey, propValue); - pToAdd.encrypted(true); - accessPoint.properties().add(pToAdd); - } - - RegistryPublisher publisher = RegistryPublisherFactory.create(); - ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); - publisher.update(serviceEndpoint); - } - } catch (Exception e) { - logger.error(e.getLocalizedMessage()); - throw e; - } - } - public String[] getActiveVREs(boolean testMode) { DatabaseManager dbaseManager = new DatabaseManager(); return dbaseManager.getActiveVres(); diff --git a/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java index f5b690a..ca45e56 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/DataMethodDownloadHarvester.java @@ -1,81 +1,58 @@ package org.gcube.dataharvest.harvester; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.text.DateFormat; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; +import java.util.List; -import org.gcube.dataharvest.AccountingDataHarvesterPlugin; -import org.gcube.dataharvest.datamodel.Harvest; -import org.gcube.dataharvest.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.commons.lang.Validate; import org.gcube.common.homelibrary.home.Home; import org.gcube.common.homelibrary.home.HomeLibrary; import org.gcube.common.homelibrary.home.HomeManager; -import org.gcube.common.homelibrary.home.exceptions.HomeNotFoundException; import org.gcube.common.homelibrary.home.exceptions.InternalErrorException; -import org.gcube.common.homelibrary.home.exceptions.UserNotFoundException; import org.gcube.common.homelibrary.home.workspace.WorkspaceItem; import org.gcube.common.homelibrary.home.workspace.accounting.AccountingEntry; -import org.gcube.common.homelibrary.home.workspace.exceptions.InsufficientPrivilegesException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemAlreadyExistException; -import org.gcube.common.homelibrary.home.workspace.exceptions.ItemNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WorkspaceFolderNotFoundException; -import org.gcube.common.homelibrary.home.workspace.exceptions.WrongDestinationException; import org.gcube.common.homelibrary.jcr.repository.JCRRepository; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspace; import org.gcube.common.homelibrary.jcr.workspace.JCRWorkspaceItem; -import org.gcube.common.homelibrary.model.exceptions.RepositoryException; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataharvest.datamodel.Harvest; +import org.gcube.dataharvest.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataMethodDownloadHarvester extends BasicHarvester { - private Date startDate, endDate; - private DateFormat format; - private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu"; + private static Logger logger = LoggerFactory.getLogger(DataMethodDownloadHarvester.class); + + private final String SCOPE = "/d4science.research-infrastructures.eu/gCubeApps/SoBigData.eu"; + private int count = 0; - public DataMethodDownloadHarvester(String start, String end) throws ParseException { - super(start, end); - //ScopeProvider.instance.set(SCOPE); - } - public DataMethodDownloadHarvester(Date start, Date end) throws ParseException { super(start, end); - //ScopeProvider.instance.set(SCOPE); } @Override public List getData() throws Exception { ArrayList data = new ArrayList(); + ScopeProvider.instance.set(SCOPE); + String[] vres = getActiveVREs(true); for (String vre : vres) { try { count = 0; - logger.error("HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager()"); - HomeManager manager = HomeLibrary.getHomeManagerFactory().getHomeManager(); String user = getVREName(vre) + "-Manager"; - logger.error("Home home = manager.getHome(user)"); - Home home = manager.getHome(user); - logger.error("JCRWorkspace ws = (JCRWorkspace) home.getWorkspace()"); - JCRWorkspace ws = (JCRWorkspace) home.getWorkspace(); - logger.error("ws.getItemByPath..."); - JCRWorkspaceItem item = (JCRWorkspaceItem) ws .getItemByPath("/Workspace/MySpecialFolders/" + getVREName(vre)); diff --git a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java index 54fa7fe..5a9784e 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/MethodInvocationHarvester.java @@ -3,25 +3,20 @@ package org.gcube.dataharvest.harvester; import java.text.ParseException; import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; -import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.analytics.Filter; import org.gcube.accounting.analytics.Info; import org.gcube.accounting.analytics.TemporalConstraint; import org.gcube.accounting.analytics.TemporalConstraint.AggregationMode; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQuery; import org.gcube.accounting.analytics.persistence.AccountingPersistenceQueryFactory; +import org.gcube.accounting.datamodel.aggregation.AggregatedServiceUsageRecord; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; - -import java.util.SortedMap; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.Iterator; - import org.gcube.dataharvest.datamodel.Harvest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,10 +25,6 @@ public class MethodInvocationHarvester extends BasicHarvester { private static Logger logger = LoggerFactory.getLogger(MethodInvocationHarvester.class); - public MethodInvocationHarvester(String start, String end) throws ParseException { - super(start, end); - } - public MethodInvocationHarvester(Date start, Date end) throws ParseException { super(start, end); } @@ -83,33 +74,6 @@ public class MethodInvocationHarvester extends BasicHarvester { logger.error(errorMessage); throw x; } - }/**/ - - /* used only for test */ - public static void main(String[] args) { - String ROOT = "18fed2d9-030b-4c77-93af-af2015d945f7-843339462"; - String GCUBE = "32542bb3-b04b-4112-9af1-7fac049d33cd-98187548"; - String TAGME = "308cf6bd-3cbb-4267-a300-0cf5ae122bc9-843339462"; - String TAGME2 = "9684c4a6-e542-44e1-a39c-4bdcf04befa8-843339462"; - String SOBIGDATA_EU = "6af6ee16-e357-4d2d-ba14-2c5984ab4e02-843339462"; - try { - MethodInvocationHarvester m = new MethodInvocationHarvester("2018-03-01 00:00:00", "2018-03-31 23:59:59"); - MethodInvocationHarvester.setContext(SOBIGDATA_EU); - System.out.println(MethodInvocationHarvester.getCurrentContext()); - Properties props = m.readServiceEndpoint(); - System.out.println("Found " + props.size() + " properties."); - Set keys = props.keySet(); - for(Object key : keys) { - System.out.println((String)key + " :: " + props.getProperty((String) key)); - } - - m.getData(); - } catch (Exception e) { - // TODO Auto-generated catch block - System.out.println(e.getLocalizedMessage()); - e.printStackTrace(); - } } - /**/ } diff --git a/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java index d16517c..a404c76 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/ResourceCatalogueHarvester.java @@ -1,127 +1,116 @@ package org.gcube.dataharvest.harvester; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.text.DateFormat; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; -import java.util.List; import java.util.Iterator; -import java.util.Locale; -import java.util.Properties; +import java.util.List; -import org.apache.http.client.utils.URLEncodedUtils; import org.gcube.dataharvest.datamodel.Harvest; import org.gcube.dataharvest.utils.Utils; import org.gcube.portlets.user.urlshortener.UrlEncoderUtil; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.quartz.utils.StringKeyDirtyFlagMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ResourceCatalogueHarvester extends BasicHarvester { + private static Logger logger = LoggerFactory.getLogger(ResourceCatalogueHarvester.class); - private int cityOfCitizensCounter = 0, migrationStudiesCounter = 0, societalDebatesCounter = 0, - wellBeingAndEconomyCounter = 0; - - public ResourceCatalogueHarvester(String start, String end) throws ParseException { - super(start, end); - } - + + private int cityOfCitizensCounter = 0; + private int migrationStudiesCounter = 0; + private int societalDebatesCounter = 0; + private int wellBeingAndEconomyCounter = 0; + public ResourceCatalogueHarvester(Date start, Date end) throws ParseException { super(start, end); } - + @Override public List getData() throws Exception { + ArrayList data = new ArrayList(); List dataDeliverable = getDataDeliverable(); - for (Harvest harvest : dataDeliverable) { + for(Harvest harvest : dataDeliverable) { data.add(harvest); } - + List dataMethod = getDataMethod(); - for (Harvest harvest : dataMethod) { + for(Harvest harvest : dataMethod) { data.add(harvest); } - + List dataDataset = getDataDataset(); - for (Harvest harvest : dataDataset) { + for(Harvest harvest : dataDataset) { data.add(harvest); } - + List dataApplication = getDataApplication(); - for (Harvest harvest : dataApplication) { + for(Harvest harvest : dataApplication) { data.add(harvest); } - + return data; } - + public List getDataDeliverable() throws Exception { String json = executeQuery("Deliverable"); return buildList(json, Harvest.NEW_CATALOGUE_DELIVERABLES); } - + public List getDataMethod() throws Exception { String json = executeQuery("Method"); return buildList(json, Harvest.NEW_CATALOGUE_METHODS); } - + public List getDataDataset() throws Exception { String json = executeQuery("Dataset"); return buildList(json, Harvest.NEW_CATALOGUE_DATASETS); } - + public List getDataApplication() throws Exception { String json = executeQuery("Application"); return buildList(json, Harvest.NEW_CATALOGUE_APPLICATIONS); } - + private List buildList(String json, int dataType) throws Exception { ArrayList data = new ArrayList(); JSONObject jsonObject = new JSONObject(json); - + JSONObject responseHeader = jsonObject.getJSONObject("responseHeader"); int status = responseHeader.getInt("status"); - if (status != 0) { + if(status != 0) { String err = "Query Deliverable in error: status " + status; logger.error(err); throw new Exception(err, null); } - + JSONObject response = jsonObject.getJSONObject("response"); - + int numFound = response.getInt("numFound"); Harvest h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue", numFound); logger.debug(h.toString()); data.add(h); - if (numFound > 0) { - + if(numFound > 0) { + JSONArray docs = response.getJSONArray("docs"); - for (Object item : docs) { + for(Object item : docs) { JSONObject doc = (JSONObject) item; try { JSONArray groups = doc.getJSONArray("groups"); Iterator git = groups.iterator(); - while (git.hasNext()) { + while(git.hasNext()) { String groupItem = (String) git.next(); counterByGroup(groupItem); } - } catch (JSONException x) { + } catch(JSONException x) { logger.debug("Document without groups"); } } - + h = new Harvest(dataType, "/d4science.research-infrastructures.eu/SoBigData/CityOfCitizens", cityOfCitizensCounter); logger.debug(h.toString()); @@ -137,13 +126,13 @@ public class ResourceCatalogueHarvester extends BasicHarvester { wellBeingAndEconomyCounter); logger.debug(h.toString()); data.add(h); - + } - + return data; - + } - + private String executeQuery(String fqSubString) throws Exception { String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?"; String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO " @@ -152,49 +141,29 @@ public class ResourceCatalogueHarvester extends BasicHarvester { String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\""); query += "&fq=" + fq + "&wt=json&indent=true"; logger.debug(query); - + String json = Utils.getJson(query); // logger.debug(json); - + return json; } - + private void counterByGroup(String groupName) { cityOfCitizensCounter = migrationStudiesCounter = societalDebatesCounter = wellBeingAndEconomyCounter = 0; - switch (groupName) { - case "city-of-citizens-group": - cityOfCitizensCounter++; - break; - case "migration-studies": - migrationStudiesCounter++; - break; - case "societal-debates-group": - societalDebatesCounter++; - break; - case "well-being-and-economy-group": - wellBeingAndEconomyCounter++; - break; + switch(groupName) { + case "city-of-citizens-group": + cityOfCitizensCounter++; + break; + case "migration-studies": + migrationStudiesCounter++; + break; + case "societal-debates-group": + societalDebatesCounter++; + break; + case "well-being-and-economy-group": + wellBeingAndEconomyCounter++; + break; } } - /* - * used only for test public static void main(String[] args) { Calendar from - * = Calendar.getInstance(); from.set(Calendar.DAY_OF_MONTH, 1); - * from.set(Calendar.MONTH, Calendar.NOVEMBER); from.set(Calendar.YEAR, - * 2017); from.set(Calendar.HOUR_OF_DAY, 0); from.set(Calendar.MINUTE, 0); - * from.set(Calendar.SECOND, 0); - * - * Calendar to = Calendar.getInstance(); to.set(Calendar.DAY_OF_MONTH, 30); - * to.set(Calendar.MONTH, Calendar.NOVEMBER); to.set(Calendar.YEAR, 2017); - * to.set(Calendar.HOUR_OF_DAY, 23); to.set(Calendar.MINUTE, 59); - * to.set(Calendar.SECOND, 59); - * - * try { ResourceCatalogueHarvester a = new - * ResourceCatalogueHarvester(from.getTime(), to.getTime()); List l - * = a.getData(); for(Harvest h : l) { System.out.println(h.toString()); } - * // System.out.println(Utils.dateToString(a.startDate)); // - * System.out.println(Utils.dateToStringWithTZ(a.endDate)); } catch - * (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } - * } - */ } diff --git a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java index d6492c5..00abb8a 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/SocialHarvester.java @@ -16,32 +16,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SocialHarvester extends BasicHarvester { - private final String CATEGORY_NAME = "Accounting"; - private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class); + + private static Logger logger = LoggerFactory.getLogger(SocialHarvester.class); + private int likes, replies, posts; private long from = 0, to = 0; - - public SocialHarvester(String start, String end) throws ParseException { - super(start, end); - this.from = startDate.getTime(); - this.to = endDate.getTime(); - } - + public SocialHarvester(Date start, Date end) throws ParseException { super(start, end); this.from = startDate.getTime(); this.to = endDate.getTime(); } - + @Override public List getData() throws Exception { ArrayList data = new ArrayList(); - Properties props = readServiceEndpoint(); String[] vres = getActiveVREs(true); - for (String vre : vres) { + for(String vre : vres) { try { logger.debug("Working on VRE: " + vre); - + String key = vre; // vre.substring(vre.lastIndexOf("/") + 1); String token = props.getProperty(key); //System.out.println(key + ":" + token); @@ -50,7 +44,7 @@ public class SocialHarvester extends BasicHarvester { Harvest harvest = new Harvest(Harvest.SOCIAL_LIKES, vre, likes); data.add(harvest); logger.debug(harvest.toString()); - + harvest = new Harvest(Harvest.SOCIAL_POSTS, vre, posts); data.add(harvest); logger.debug(harvest.toString()); @@ -58,52 +52,41 @@ public class SocialHarvester extends BasicHarvester { harvest = new Harvest(Harvest.SOCIAL_REPLIES, vre, replies); data.add(harvest); logger.debug(harvest.toString()); - } catch (Exception x) { + } catch(Exception x) { logger.error("SocialHarvester::getJson. " + vre + ". " + x.getLocalizedMessage()); } } return data; } - + private void getJson(String token) throws MalformedURLException, IOException { likes = replies = posts = 0; // la seguente stringa deve essere letta dinamicamente String url = "https://socialnetworking1.d4science.org/social-networking-library-ws/rest/2/posts/get-posts-vre?gcube-token="; JSONObject jsonObject = new JSONObject(Utils.getJson(url + token)); - + Boolean success = (Boolean) jsonObject.get("success"); - if (success == false) { + if(success == false) { String message = "getJson() on token: " + token + " success=false"; logger.error(message); throw new IOException(message); } - + JSONArray res = jsonObject.getJSONArray("result"); int len = res.length(); - for (int i = 0; i < len; i++) { + for(int i = 0; i < len; i++) { JSONObject item = res.getJSONObject(i); long time = item.getLong("time"); //System.out.println(from + " - " + time + " - " + to ); - if ((from <= time) && (time <= to)) { + if((from <= time) && (time <= to)) { posts++; replies += item.getInt("comments_no"); likes += item.getInt("likes_no"); } } } + -/**/ - public static void main(String[] argv) { - try { - SocialHarvester a = new SocialHarvester("2018-01-01 00:00:00", "2018-01-31 23:59:59"); - List list = a.getData(); - for (Harvest l : list) { - System.out.println(l.toString()); - } - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - }/**/ + } diff --git a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java index 77c611f..25f32d6 100644 --- a/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java +++ b/src/main/java/org/gcube/dataharvest/harvester/VreUsersHarvester.java @@ -15,13 +15,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class VreUsersHarvester extends BasicHarvester { - // private final String CATEGORY_NAME = "Accounting"; + + private static Logger logger = LoggerFactory.getLogger(VreUsersHarvester.class); - public VreUsersHarvester(String start, String end) throws ParseException { - super(start, end); - } - public VreUsersHarvester(Date start, Date end) throws ParseException { super(start, end); } @@ -60,63 +57,5 @@ public class VreUsersHarvester extends BasicHarvester { userNumber = jsonObject.getJSONArray("result").length(); return userNumber; } - - // private Properties readServiceEndpoint() throws Exception { - // Properties props = new Properties(); - // String scope = "/d4science.research-infrastructures.eu"; - // ScopeProvider.instance.set("/d4science.research-infrastructures.eu"); - // - // SimpleQuery query = queryFor(ServiceEndpoint.class); - // query.addCondition("$resource/Profile/Name/text() eq '" + - // RUNTIME_RESOURCE_NAME + "'"); - // query.addCondition("$resource/Profile/Category/text() eq '" + - // CATEGORY_NAME + "'"); - // - // DiscoveryClient client = - // clientFor(ServiceEndpoint.class); - // String password = null; - // try { - // List list = client.submit(query); - // if (list.size() > 1) { - // logger.error("Too many Service Endpoints having name " + - // RUNTIME_RESOURCE_NAME - // + " in this scope having Category " + CATEGORY_NAME); - // } else if (list.size() == 0) { - // logger.error("There is no Service Endpoint having name " + - // RUNTIME_RESOURCE_NAME - // + " and Category " + CATEGORY_NAME + " in this scope " + scope); - // } else { - // for (ServiceEndpoint res : list) { - // AccessPoint[] accessPoints = (AccessPoint[]) res.profile().accessPoints() - // .toArray(new AccessPoint[res.profile().accessPoints().size()]); - // for (AccessPoint found : accessPoints) { - // password = - // org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(found.password()); - // for (ServiceEndpoint.Property prop : found.properties()) { - // props.setProperty(prop.name(), - // org.gcube.common.encryption.StringEncrypter.getEncrypter().decrypt(prop.value())); - // } - // } - // - // } - // } - // return props; - // } catch (Exception e) { - // logger.error(e.getLocalizedMessage()); - // throw e; - // } - // } - - // public static void main(String[] argv) { - // try { - // VreUsersHarvester a = new VreUsersHarvester("2017-01-01", "2017-02-01"); - // List list = a.getData(); - // for(Harvest l : list) { - // System.out.println(l.toString()); - // } - // } catch (Exception e) { - // // TODO Auto-generated catch block - // e.printStackTrace(); - // } - // } + } diff --git a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java index c839d56..9d1f70b 100644 --- a/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java +++ b/src/main/java/org/gcube/dataharvest/utils/ContextAuthorization.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; +import java.util.TreeMap; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.UserInfo; @@ -41,12 +42,14 @@ public class ContextAuthorization { */ protected Map tokenToContext; + + + /** * Contains Properties used to generate tokens */ - public ContextAuthorization() throws Exception { - this.contextToToken = new HashMap<>(); + this.contextToToken = new TreeMap<>(); this.tokenToContext = new HashMap<>(); retrieveContextsAndTokens(); } diff --git a/src/main/java/org/gcube/dataharvest/utils/Utils.java b/src/main/java/org/gcube/dataharvest/utils/Utils.java index 4a0eb9f..15be59b 100644 --- a/src/main/java/org/gcube/dataharvest/utils/Utils.java +++ b/src/main/java/org/gcube/dataharvest/utils/Utils.java @@ -25,10 +25,6 @@ public class Utils { return cal; } - public static String dateToString(Date date) { - DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); - return formatter.format(date); - } public static String dateToStringWithTZ(Date date) { DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");