Code redesign
Refs #11756: Refactor DataHArvesterPlugin to support scheduled execution from smart-executor Task-Url: https://support.d4science.org/issues/11756 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/accounting/accounting-dashboard-harvester-se-plugin@167601 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
3c4c1a6b93
commit
5de25d5a2b
|
@ -8,6 +8,7 @@ import java.util.List;
|
||||||
|
|
||||||
import org.gcube.dataharvest.datamodel.HarvestedData;
|
import org.gcube.dataharvest.datamodel.HarvestedData;
|
||||||
import org.gcube.dataharvest.harvester.BasicHarvester;
|
import org.gcube.dataharvest.harvester.BasicHarvester;
|
||||||
|
import org.gcube.dataharvest.utils.DateUtils;
|
||||||
import org.gcube.dataharvest.utils.Utils;
|
import org.gcube.dataharvest.utils.Utils;
|
||||||
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
|
import org.gcube.portlets.user.urlshortener.UrlEncoderUtil;
|
||||||
import org.json.JSONArray;
|
import org.json.JSONArray;
|
||||||
|
@ -140,8 +141,8 @@ public class ResourceCatalogueHarvester extends BasicHarvester {
|
||||||
|
|
||||||
private String executeQuery(String fqSubString) throws Exception {
|
private String executeQuery(String fqSubString) throws Exception {
|
||||||
String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?";
|
String query = "https://ckan-solr-d4s.d4science.org/solr/sobigdata/select?";
|
||||||
String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + Utils.dateToStringWithTZ(startDate) + " TO "
|
String q = UrlEncoderUtil.encodeQuery("metadata_created:[" + DateUtils.dateToStringWithTZ(startDate) + " TO "
|
||||||
+ Utils.dateToStringWithTZ(endDate) + "]");
|
+ DateUtils.dateToStringWithTZ(endDate) + "]");
|
||||||
query += "q=" + q;
|
query += "q=" + q;
|
||||||
String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\"");
|
String fq = UrlEncoderUtil.encodeQuery("extras_systemtype:\"SoBigData.eu: " + fqSubString + "\"");
|
||||||
query += "&fq=" + fq + "&wt=json&indent=true";
|
query += "&fq=" + fq + "&wt=json&indent=true";
|
||||||
|
|
|
@ -10,6 +10,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* @author Eric Perrone (ISTI - CNR)
|
||||||
* @author Luca Frosini (ISTI - CNR)
|
* @author Luca Frosini (ISTI - CNR)
|
||||||
*/
|
*/
|
||||||
public class DateUtils {
|
public class DateUtils {
|
||||||
|
@ -97,6 +98,8 @@ public class DateUtils {
|
||||||
return aggregationEndDate.getTime();
|
return aggregationEndDate.getTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* OLD functions of Eric Perrone (ISTI - CNR) */
|
||||||
|
|
||||||
public static Calendar dateToCalendar(Date date) {
|
public static Calendar dateToCalendar(Date date) {
|
||||||
Calendar cal = null;
|
Calendar cal = null;
|
||||||
cal = Calendar.getInstance();
|
cal = Calendar.getInstance();
|
||||||
|
@ -104,4 +107,8 @@ public class DateUtils {
|
||||||
return cal;
|
return cal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String dateToStringWithTZ(Date date) {
|
||||||
|
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
||||||
|
return formatter.format(date) + "Z";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,9 +6,6 @@ import java.io.InputStreamReader;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.text.DateFormat;
|
|
||||||
import java.text.SimpleDateFormat;
|
|
||||||
import java.util.Date;
|
|
||||||
|
|
||||||
import org.gcube.common.authorization.client.Constants;
|
import org.gcube.common.authorization.client.Constants;
|
||||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||||
|
@ -26,11 +23,6 @@ public class Utils {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
private static Logger logger = LoggerFactory.getLogger(Utils.class);
|
||||||
|
|
||||||
public static String dateToStringWithTZ(Date date) {
|
|
||||||
DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
|
||||||
return formatter.format(date) + "Z";
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getJson(String url) throws MalformedURLException, IOException {
|
public static String getJson(String url) throws MalformedURLException, IOException {
|
||||||
URL address = new URL(url);
|
URL address = new URL(url);
|
||||||
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
|
HttpURLConnection connection = (HttpURLConnection) address.openConnection();
|
||||||
|
|
Loading…
Reference in New Issue