231 lines
6.6 KiB
Java
231 lines
6.6 KiB
Java
package org.gcube.dataharvest;
|
|
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.List;
|
|
|
|
import org.gcube.dataharvest.dao.Dao;
|
|
import org.gcube.dataharvest.dao.DaoException;
|
|
import org.gcube.dataharvest.dao.DatabaseConnectionData;
|
|
import org.gcube.dataharvest.dao.DatabaseParameterRetriever;
|
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
|
import org.gcube.dataharvest.harvester.BasicHarvester;
|
|
import org.gcube.dataharvest.harvester.SocialHarvester;
|
|
import org.gcube.dataharvest.harvester.VREUsersHarvester;
|
|
import org.gcube.dataharvest.harvester.sobigdata.DataMethodDownloadHarvester;
|
|
import org.gcube.dataharvest.harvester.sobigdata.TagMeMethodInvocationHarvester;
|
|
import org.gcube.dataharvest.harvester.sobigdata.ResourceCatalogueHarvester;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
/**
|
|
* @author Eric Perrone (ISTI - CNR)
|
|
* @author Luca Frosini (ISTI - CNR)
|
|
*/
|
|
public class Harvester {
|
|
|
|
private static Logger logger = LoggerFactory.getLogger(Harvester.class);
|
|
|
|
public static final String PARAMETER_FROM = "from";
|
|
public static final String PARAMETER_TO = "to";
|
|
public static final String TEST = "test";
|
|
private boolean testMode = false;
|
|
private Date dateFrom, dateTo;
|
|
|
|
public static void main(String[] args) {
|
|
Harvester harvester = new Harvester();
|
|
try {
|
|
harvester.processParameterArray(args);
|
|
SocialHarvester socialHarvester = new SocialHarvester(harvester.getDateFrom(), harvester.getDateTo());
|
|
harvester.runOne(socialHarvester);
|
|
// harvester.run();
|
|
//ArrayList<Integer> list = harvester.getSubTree(17);
|
|
//harvester.createSocialReports(list);
|
|
System.out.println("End.");
|
|
|
|
} catch(Exception e) {
|
|
// TODO Auto-generated catch block
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
public Harvester() {
|
|
testMode = false;
|
|
}
|
|
|
|
public void setDateFrom(Date dateFrom) {
|
|
this.dateFrom = dateFrom;
|
|
}
|
|
|
|
public void setDateTo(Date dateTo) {
|
|
this.dateTo = dateTo;
|
|
}
|
|
|
|
public void setTestMode(boolean testMode) {
|
|
this.testMode = testMode;
|
|
}
|
|
|
|
public Date getDateFrom() {
|
|
return dateFrom;
|
|
}
|
|
|
|
public Date getDateTo() {
|
|
return dateTo;
|
|
}
|
|
|
|
public void runOne(BasicHarvester harvester) {
|
|
try {
|
|
List<HarvestedData> data = harvester.getData();
|
|
if(data != null) {
|
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, data);
|
|
}
|
|
} catch(Exception x) {
|
|
logger.error(x.getLocalizedMessage());
|
|
}
|
|
}
|
|
|
|
public void runAll() {
|
|
try {
|
|
// collecting info on VRE users
|
|
VREUsersHarvester vreUsersHarvester = new VREUsersHarvester(dateFrom, dateTo);
|
|
List<HarvestedData> users = vreUsersHarvester.getData();
|
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, users);
|
|
} catch(Exception x) {
|
|
logger.error(x.getLocalizedMessage());
|
|
}
|
|
|
|
try {
|
|
// collecting info on Res. Catalogue (Dataset, Application,
|
|
// Deliverables, Methods)
|
|
ResourceCatalogueHarvester resourceCatalogueHarvester = new ResourceCatalogueHarvester(dateFrom, dateTo);
|
|
List<HarvestedData> res = resourceCatalogueHarvester.getData();
|
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
|
} catch(Exception x) {
|
|
logger.error(x.getLocalizedMessage());
|
|
}
|
|
|
|
try {
|
|
// collecting info on Data/Method download
|
|
DataMethodDownloadHarvester dataMethodDownloadHarvester = new DataMethodDownloadHarvester(dateFrom, dateTo);
|
|
List<HarvestedData> res = dataMethodDownloadHarvester.getData();
|
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
|
} catch(Exception x) {
|
|
logger.error(x.getLocalizedMessage());
|
|
}
|
|
|
|
try {
|
|
// collecting info on social (posts, replies and likes)
|
|
SocialHarvester socialHarvester = new SocialHarvester(dateFrom, dateTo);
|
|
List<HarvestedData> res = socialHarvester.getData();
|
|
insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
|
} catch(Exception x) {
|
|
logger.error(x.getLocalizedMessage());
|
|
}
|
|
|
|
try {
|
|
// collecting info on method invocation
|
|
TagMeMethodInvocationHarvester methodInvocationHarvester = new TagMeMethodInvocationHarvester(dateFrom, dateTo);
|
|
List<HarvestedData> res = methodInvocationHarvester.getData();
|
|
logger.debug("{}", res);
|
|
// insertMonthlyData((Date) dateFrom, (Date) dateTo, res);
|
|
} catch(Exception x) {
|
|
logger.error(x.getLocalizedMessage());
|
|
}
|
|
}
|
|
|
|
private void insertMonthlyData(Date from, Date to, List<HarvestedData> data) {
|
|
Dao dao = null;
|
|
try {
|
|
dao = dbConnect();
|
|
dao.insertMonthlyMeasure(data, from, to, false);
|
|
} catch(Exception e) {
|
|
logger.error(e.getLocalizedMessage());
|
|
} finally {
|
|
if(dao != null) {
|
|
try {
|
|
dao.disconnect();
|
|
} catch(DaoException e) {
|
|
logger.error(e.getLocalizedMessage());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private Dao dbConnect() throws DaoException {
|
|
DatabaseParameterRetriever dde = new DatabaseParameterRetriever();
|
|
//dde.setTestMode(testMode);
|
|
DatabaseConnectionData dcd = dde.retrieveDatabaseInfo();
|
|
Dao dao = new Dao();
|
|
dao.init();
|
|
dao.connect(dcd.getURI(), dcd.getUser(), dcd.getPassword());
|
|
return dao;
|
|
}
|
|
|
|
private void processParameterArray(String[] args) throws Exception {
|
|
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
|
for(int i = 0; i < args.length; i++) {
|
|
if(args[i].startsWith(PARAMETER_FROM)) {
|
|
String from = getArgValue(args[i]);
|
|
dateFrom = formatter.parse(from);
|
|
System.out.println(dateFrom.toString());
|
|
} else if(args[i].startsWith(PARAMETER_TO)) {
|
|
String to = getArgValue(args[i]);
|
|
dateTo = formatter.parse(to);
|
|
System.out.println(dateTo.toString());
|
|
} else if(args[i].toLowerCase().startsWith(TEST)) {
|
|
testMode = true;
|
|
System.out.println("Test: " + testMode);
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
private String getArgValue(String arg) throws Exception {
|
|
String[] tokens = arg.split("=");
|
|
if(tokens.length != 2)
|
|
throw new Exception("Argument must be in the format 'name=value'.");
|
|
return tokens[1].trim();
|
|
}
|
|
|
|
protected ArrayList<Integer> getSubTree(int root) {
|
|
Dao dao = null;
|
|
try {
|
|
dao = dbConnect();
|
|
ArrayList<Integer> subTree = dao.getSubTree(root);
|
|
return subTree;
|
|
} catch(Exception e) {
|
|
logger.error(e.getLocalizedMessage());
|
|
return null;
|
|
} finally {
|
|
if(dao != null) {
|
|
try {
|
|
dao.disconnect();
|
|
} catch(DaoException e) {
|
|
logger.error(e.getLocalizedMessage());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
protected void createSocialReports(ArrayList<Integer> ids) {
|
|
Dao dao = null;
|
|
try {
|
|
dao = dbConnect();
|
|
for(Integer contextId : ids)
|
|
dao.createSocialReport(contextId, 2018);
|
|
} catch(Exception e) {
|
|
logger.error(e.getLocalizedMessage());
|
|
} finally {
|
|
if(dao != null) {
|
|
try {
|
|
dao.disconnect();
|
|
} catch(DaoException e) {
|
|
logger.error(e.getLocalizedMessage());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|