From bcf591cc8c528bc3f90707586ab1fef5baa17c80 Mon Sep 17 00:00:00 2001 From: Gianpaolo Coro Date: Mon, 24 Feb 2014 11:52:52 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@92158 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../dataanalysis/geo/batch/Downloader.java | 122 ++++++++++++++++++ .../geo/batch/EnvriMetadataInsertDev.java | 10 +- 2 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysis/geo/batch/Downloader.java diff --git a/src/main/java/org/gcube/dataanalysis/geo/batch/Downloader.java b/src/main/java/org/gcube/dataanalysis/geo/batch/Downloader.java new file mode 100644 index 0000000..0249ec1 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/geo/batch/Downloader.java @@ -0,0 +1,122 @@ +package org.gcube.dataanalysis.geo.batch; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.Writer; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.List; +/** + * Analyzes each row of a file and searches for links inside the file + * In the end builds a wget file to be run + * @author coro + * + */ +public class Downloader { + + public static void main(String[] args) throws Exception { + List hfiles = getfiles("netcdf_data.html", "fileServer","http", ".nc"); + System.out.println(hfiles); + System.out.println("Number of links:"+hfiles.size()); + List files = enrichfiles(hfiles); + System.out.println(files); + buildwgetFile("wgetfiles.sh", hfiles, files); + } + + public static void buildwgetFile(String filename, List hfiles, List files) throws Exception{ + int size = hfiles.size(); + BufferedWriter bw = new BufferedWriter(new FileWriter(new File(filename))); + for (int i=0;i enrichfiles(List files) throws Exception{ + List arrayfile = new ArrayList(); + for (String fileh:files){ + String file = fileh.substring(fileh.lastIndexOf("/")+1); + if (file.contains("temperature")) + file = buildTopicString(file,"ENVIRONMENT", "OCEANS"); + else if (file.contains("salinity")) + file = buildTopicString(file,"ENVIRONMENT", "OCEANS"); + else if (file.contains("oxygen")) + file = buildTopicString(file,"ENVIRONMENT", "BIOTA"); + else if (file.contains("phosphate")) + file = buildTopicString(file,"ENVIRONMENT", "BIOTA"); + else if (file.contains("nitrate")) + file = buildTopicString(file,"ENVIRONMENT", "BIOTA"); + else if (file.contains("silicate")) + file = buildTopicString(file,"ENVIRONMENT", "BIOTA"); + else + file = buildTopicString(file,"ENVIRONMENT", "OCEANS"); + arrayfile.add(file); + } + return arrayfile; + } + + public static List getfiles(String filename,String criterion,String initselection, String endselection) throws Exception{ + List files = new ArrayList(); + BufferedReader br = new BufferedReader(new FileReader(new File(filename))); + String line = br.readLine(); + while (line!=null){ + if (line.contains(criterion)){ + String cut = line.substring(line.indexOf(initselection),line.indexOf(endselection)+endselection.length()); + files.add(cut); + } + line = br.readLine(); + } + br.close(); + return files; + } + public static String buildGetterString(String httpstring,String filename){ + return String.format("wget --output-document=%1$s %2$s",filename,httpstring); + } + + public static String buildTopicString(String filename, String... topics){ + int idx = filename.indexOf(".n"); + String file = filename.substring(0,idx); + String ext = filename.substring(idx); + for (String topic:topics){ + file=file+"_"+topic; + } + return file+"_"+ext; + } + + public static void downloadData(String endpoint, String file) throws Exception { + // Send data + String urlStr = endpoint; + URL url = new URL(urlStr); + URLConnection conn = url.openConnection(); + InputStreamReader isr = new InputStreamReader(conn.getInputStream()); + FileWriter fw = new FileWriter(new File(file)); + pipe(isr, fw); + fw.close(); + isr.close(); + } + + private static void pipe(Reader reader, Writer writer) throws IOException { + char[] buf = new char[1024]; + int read = 0; + double bytes = 0; + long i = 0; + while ((read = reader.read(buf)) >= 0) { + writer.write(buf, 0, read); + bytes=(bytes+read); + if (i%1000==0) + System.out.println("B:"+bytes); + i++; + } + + writer.flush(); + } + +} diff --git a/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java b/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java index 5ab375e..5b2a9c5 100644 --- a/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java +++ b/src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java @@ -5,9 +5,9 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import org.gcube.dataanalysis.geo.connectors.netcdf.NetCDFDataExplorer; import org.gcube.dataanalysis.geo.meta.GenericLayerMetadata; import org.gcube.dataanalysis.geo.meta.OGCFormatter; -import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer; import org.opengis.metadata.identification.TopicCategory; import ucar.nc2.dataset.CoordinateAxis; @@ -86,10 +86,10 @@ public class EnvriMetadataInsertDev { List gridTypes = gds.getGrids(); GridDatatype gdt = gridTypes.get(0); - double minX = ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem()); - double maxX = ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem()); - double minY = ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem()); - double maxY = ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem()); + double minX = NetCDFDataExplorer.getMinX(gdt.getCoordinateSystem()); + double maxX = NetCDFDataExplorer.getMaxX(gdt.getCoordinateSystem()); + double minY = NetCDFDataExplorer.getMinY(gdt.getCoordinateSystem()); + double maxY = NetCDFDataExplorer.getMaxY(gdt.getCoordinateSystem()); System.out.println("minX: "+minX+" minY: "+minY+" maxX:"+maxX+" maxY:"+maxY);