gCubeSystem
/
ecological-engine-geospatial-extensions
16
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/EcologicalEngineGeoSpatialExtension@92158 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Gianpaolo Coro 2014-02-24 11:52:52 +00:00
parent b5892c36a4
commit bcf591cc8c
2 changed files with 127 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
src/main/java/org/gcube/dataanalysis/geo/batch/Downloader.java Normal file
View File

@ -0,0 +1,122 @@
package org.gcube.dataanalysis.geo.batch;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
/**
* Analyzes each row of a file and searches for links inside the file
* In the end builds a wget file to be run
* @author coro
*
*/
public class Downloader {
public static void main(String[] args) throws Exception {
List<String> hfiles = getfiles("netcdf_data.html", "fileServer","http", ".nc");
System.out.println(hfiles);
System.out.println("Number of links:"+hfiles.size());
List<String> files = enrichfiles(hfiles);
System.out.println(files);
buildwgetFile("wgetfiles.sh", hfiles, files);
}
public static void buildwgetFile(String filename, List<String> hfiles, List<String> files) throws Exception{
int size = hfiles.size();
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(filename)));
for (int i=0;i<size;i++){
bw.write(buildGetterString(hfiles.get(i), files.get(i))+System.getProperty("line.separator"));
}
bw.close();
}
public static List<String> enrichfiles(List<String> files) throws Exception{
List<String> arrayfile = new ArrayList<String>();
for (String fileh:files){
String file = fileh.substring(fileh.lastIndexOf("/")+1);
if (file.contains("temperature"))
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
else if (file.contains("salinity"))
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
else if (file.contains("oxygen"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else if (file.contains("phosphate"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else if (file.contains("nitrate"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else if (file.contains("silicate"))
file = buildTopicString(file,"ENVIRONMENT", "BIOTA");
else
file = buildTopicString(file,"ENVIRONMENT", "OCEANS");
arrayfile.add(file);
}
return arrayfile;
}
public static List<String> getfiles(String filename,String criterion,String initselection, String endselection) throws Exception{
List<String> files = new ArrayList<String>();
BufferedReader br = new BufferedReader(new FileReader(new File(filename)));
String line = br.readLine();
while (line!=null){
if (line.contains(criterion)){
String cut = line.substring(line.indexOf(initselection),line.indexOf(endselection)+endselection.length());
files.add(cut);
}
line = br.readLine();
}
br.close();
return files;
}
public static String buildGetterString(String httpstring,String filename){
return String.format("wget --output-document=%1$s %2$s",filename,httpstring);
}
public static String buildTopicString(String filename, String... topics){
int idx = filename.indexOf(".n");
String file = filename.substring(0,idx);
String ext = filename.substring(idx);
for (String topic:topics){
file=file+"_"+topic;
}
return file+"_"+ext;
}
public static void downloadData(String endpoint, String file) throws Exception {
// Send data
String urlStr = endpoint;
URL url = new URL(urlStr);
URLConnection conn = url.openConnection();
InputStreamReader isr = new InputStreamReader(conn.getInputStream());
FileWriter fw = new FileWriter(new File(file));
pipe(isr, fw);
fw.close();
isr.close();
}
private static void pipe(Reader reader, Writer writer) throws IOException {
char[] buf = new char[1024];
int read = 0;
double bytes = 0;
long i = 0;
while ((read = reader.read(buf)) >= 0) {
writer.write(buf, 0, read);
bytes=(bytes+read);
if (i%1000==0)
System.out.println("B:"+bytes);
i++;
}
writer.flush();
}
}

10
src/main/java/org/gcube/dataanalysis/geo/batch/EnvriMetadataInsertDev.java
View File

@ -5,9 +5,9 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import org.gcube.dataanalysis.geo.connectors.netcdf.NetCDFDataExplorer;
import org.gcube.dataanalysis.geo.meta.GenericLayerMetadata;
import org.gcube.dataanalysis.geo.meta.OGCFormatter;
import org.gcube.dataanalysis.geo.utils.ThreddsDataExplorer;
import org.opengis.metadata.identification.TopicCategory;
import ucar.nc2.dataset.CoordinateAxis;
@ -86,10 +86,10 @@ public class EnvriMetadataInsertDev {
List<GridDatatype> gridTypes = gds.getGrids();
GridDatatype gdt = gridTypes.get(0);
double minX = ThreddsDataExplorer.getMinX(gdt.getCoordinateSystem());
double maxX = ThreddsDataExplorer.getMaxX(gdt.getCoordinateSystem());
double minY = ThreddsDataExplorer.getMinY(gdt.getCoordinateSystem());
double maxY = ThreddsDataExplorer.getMaxY(gdt.getCoordinateSystem());
double minX = NetCDFDataExplorer.getMinX(gdt.getCoordinateSystem());
double maxX = NetCDFDataExplorer.getMaxX(gdt.getCoordinateSystem());
double minY = NetCDFDataExplorer.getMinY(gdt.getCoordinateSystem());
double maxY = NetCDFDataExplorer.getMaxY(gdt.getCoordinateSystem());
System.out.println("minX: "+minX+" minY: "+minY+" maxX:"+maxX+" maxY:"+maxY);