package org.gcube.dataharvest; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.SortedSet; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; import org.gcube.dataharvest.dao.DatabaseManager; import org.gcube.dataharvest.datamodel.HarvestedData; import org.gcube.dataharvest.harvester.MethodInvocationHarvester; import org.gcube.dataharvest.harvester.SocialInteractionsHarvester; import org.gcube.dataharvest.harvester.VREAccessesHarvester; import org.gcube.dataharvest.harvester.VREUsersHarvester; import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester; import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester; import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester; import org.gcube.dataharvest.utils.ContextAuthorization; import org.gcube.dataharvest.utils.DateUtils; import org.gcube.dataharvest.utils.MeasureType; import org.gcube.dataharvest.utils.Utils; import org.gcube.vremanagement.executor.plugin.Plugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Eric Perrone (ISTI - CNR) * @author Luca Frosini (ISTI - CNR) */ public class AccountingDataHarvesterPlugin extends Plugin { private static Logger logger = LoggerFactory.getLogger(AccountingDataHarvesterPlugin.class); private static final String PROPERTY_FILENAME = "config.properties"; public static final String START_DATE_INPUT_PARAMETER = "startDate"; public static final String MEASURE_TYPE_INPUT_PARAMETER = "measureType"; public static final String RERUN_INPUT_PARAMETER = "reRun"; public static final String GET_VRE_USERS_INPUT_PARAMETER = "getVREUsers"; public static final String DRY_RUN_INPUT_PARAMETER = "dryRun"; public static final String SO_BIG_DATA_VO = "/d4science.research-infrastructures.eu/SoBigData"; public static final String SO_BIG_DATA_CATALOGUE_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/ResourceCatalogue"; public static final String TAGME_CONTEXT = "/d4science.research-infrastructures.eu/SoBigData/TagMe"; protected Date start; protected Date end; public AccountingDataHarvesterPlugin(DataHarvestPluginDeclaration pluginDeclaration) { super(pluginDeclaration); } private static final InheritableThreadLocal properties = new InheritableThreadLocal() { @Override protected Properties initialValue() { return new Properties(); } }; public static InheritableThreadLocal getProperties() { return properties; } public Properties getConfigParameters() throws IOException { Properties properties = new Properties(); try { InputStream input = AccountingDataHarvesterPlugin.class.getClassLoader() .getResourceAsStream(PROPERTY_FILENAME); properties.load(input); return properties; } catch(Exception e) { logger.warn( "Unable to load {} file containing configuration properties. AccountingDataHarvesterPlugin will use defaults", PROPERTY_FILENAME); } return properties; } /** {@inheritDoc} */ @Override public void launch(Map inputs) throws Exception { logger.debug("{} is starting", this.getClass().getSimpleName()); if(inputs == null || inputs.isEmpty()) { throw new IllegalArgumentException("The can only be launched providing valid input parameters"); } if(!inputs.containsKey(MEASURE_TYPE_INPUT_PARAMETER)) { throw new IllegalArgumentException("Please set required parameter '" + MEASURE_TYPE_INPUT_PARAMETER + "'"); } MeasureType measureType = MeasureType.valueOf((String) inputs.get(MEASURE_TYPE_INPUT_PARAMETER)); boolean reRun = true; if(inputs.containsKey(RERUN_INPUT_PARAMETER)) { try { reRun = (boolean) inputs.get(RERUN_INPUT_PARAMETER); }catch (Exception e) { throw new IllegalArgumentException("'" + RERUN_INPUT_PARAMETER + "' must be a boolean"); } } boolean getVREUsers = true; if(inputs.containsKey(GET_VRE_USERS_INPUT_PARAMETER)) { try { reRun = (boolean) inputs.get(GET_VRE_USERS_INPUT_PARAMETER); }catch (Exception e) { throw new IllegalArgumentException("'" + GET_VRE_USERS_INPUT_PARAMETER + "' must be a boolean"); } } boolean dryRun = true; if(inputs.containsKey(DRY_RUN_INPUT_PARAMETER)) { try { dryRun = (boolean) inputs.get(DRY_RUN_INPUT_PARAMETER); }catch (Exception e) { throw new IllegalArgumentException("'" + DRY_RUN_INPUT_PARAMETER + "' must be a boolean"); } } if(inputs.containsKey(START_DATE_INPUT_PARAMETER)) { String startDateString = (String) inputs.get(START_DATE_INPUT_PARAMETER); start = DateUtils.UTC_DATE_FORMAT.parse(startDateString + " " + DateUtils.UTC); } else { start = DateUtils.getPreviousPeriod(measureType).getTime(); } end = DateUtils.getEndDateFromStartDate(measureType, start, 1); logger.debug("Harvesting from {} to {}", DateUtils.format(start), DateUtils.format(end)); Properties properties = getConfigParameters(); getProperties().set(properties); ContextAuthorization contextAuthorization = new ContextAuthorization(); DatabaseManager dbaseManager = new DatabaseManager(); SortedSet contexts = contextAuthorization.getContexts(); ArrayList data = new ArrayList(); String initialToken = SecurityTokenProvider.instance.get(); VREAccessesHarvester vreAccessesHarvester = null; for(String context : contexts) { // Setting the token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); ScopeBean scopeBean = new ScopeBean(context); if(vreAccessesHarvester == null) { if(scopeBean.is(Type.INFRASTRUCTURE)) { vreAccessesHarvester = new VREAccessesHarvester(start, end); }else { // This code should be never used because the scopes are sorted by fullname ScopeBean parent = scopeBean.enclosingScope(); while(!parent.is(Type.INFRASTRUCTURE)) { parent = scopeBean.enclosingScope(); } // Setting back token for the context Utils.setContext(contextAuthorization.getTokenForContext(parent.toString())); vreAccessesHarvester = new VREAccessesHarvester(start, end); // Setting back token for the context Utils.setContext(contextAuthorization.getTokenForContext(context)); } } try { if(context.startsWith(SO_BIG_DATA_VO) && start.before(DateUtils.getStartCalendar(2018, Calendar.APRIL, 1).getTime())) { logger.info("Not Harvesting VREs Accesses for {} from {} to {}", context, DateUtils.format(start), DateUtils.format(end)); } else { // Collecting Google Analytics Data for VREs Accesses List harvested = vreAccessesHarvester.getData(); data.addAll(harvested); } } catch(Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } try { // Collecting info on social (posts, replies and likes) SocialInteractionsHarvester socialHarvester = new SocialInteractionsHarvester(start, end); List harvested = socialHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Social Interactions for {}", context, e); } try { // Collecting info on VRE users if(getVREUsers) { // Harvesting Users only for VREs (not for VO and ROOT which is the sum of the children contexts) // The VREUsers can be only Harvested for the lst month if(scopeBean.is(Type.VRE) && start.equals(DateUtils.getPreviousPeriod(measureType).getTime())) { VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(start, end); List harvested = vreUsersHarvester.getData(); data.addAll(harvested); } } } catch(Exception e) { logger.error("Error harvesting Context Users for {}", context, e); } if(context.startsWith(SO_BIG_DATA_CATALOGUE_CONTEXT)) { try { // Collecting info on Resource Catalogue (Dataset, Application, Deliverables, Methods) ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(start, end, contexts); List harvested = resourceCatalogueHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Resource Catalogue Information for {}", context, e); } try { // Collecting info on Data/Method download DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(start, end, contexts); List harvested = dataMethodDownloadHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Data Method Download for {}", context, e); } } if(context.startsWith(TAGME_CONTEXT)) { try { // Collecting info on method invocation TagMeMethodInvocationHarvester tagMeMethodInvocationHarvester = new TagMeMethodInvocationHarvester(start, end); List harvested = tagMeMethodInvocationHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Method Invocations for {}", context, e); } }else { try { // Collecting info on method invocation MethodInvocationHarvester methodInvocationHarvester = new MethodInvocationHarvester(start, end); List harvested = methodInvocationHarvester.getData(); data.addAll(harvested); } catch(Exception e) { logger.error("Error harvesting Method Invocations for {}", context, e); } } } Utils.setContext(initialToken); logger.debug("Harvest Measures from {} to {} are {}", DateUtils.format(start), DateUtils.format(end), data); if(!dryRun) { dbaseManager.insertMonthlyData(start, end, data, reRun); } } /** {@inheritDoc} */ @Override protected void onStop() throws Exception { logger.debug("{} is stopping", this.getClass().getSimpleName()); } }