diff --git a/.classpath b/.classpath index a51675f..2a9e516 100644 --- a/.classpath +++ b/.classpath @@ -13,7 +13,12 @@ - + + + + + + @@ -24,6 +29,10 @@ - + + + + + diff --git a/.project b/.project index c3d425a..d3ee289 100644 --- a/.project +++ b/.project @@ -5,6 +5,11 @@ + + org.eclipse.wst.common.project.facet.core.builder + + + org.eclipse.jdt.core.javabuilder @@ -15,9 +20,15 @@ + + org.eclipse.wst.validation.validationbuilder + + + org.eclipse.jdt.core.javanature org.eclipse.m2e.core.maven2Nature + org.eclipse.wst.common.project.facet.core.nature diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index e21538b..4ede96d 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,13 +1,2 @@ eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.7 diff --git a/pom.xml b/pom.xml index db30537..7c313bb 100644 --- a/pom.xml +++ b/pom.xml @@ -60,9 +60,14 @@ org.gcube.spatial.data gis-interface - 2.4.2 + [2.4.0-SNAPSHOT,3.0.0) + + org.gcube.spatial.data + geonetwork + 3.4.1-SNAPSHOT + diff --git a/src/main/java/org/gcube/application/geoportal/PostgisDBManager.java b/src/main/java/org/gcube/application/geoportal/PostgisDBManager.java index e5c1285..95a7af0 100644 --- a/src/main/java/org/gcube/application/geoportal/PostgisDBManager.java +++ b/src/main/java/org/gcube/application/geoportal/PostgisDBManager.java @@ -2,10 +2,7 @@ package org.gcube.application.geoportal; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -18,6 +15,7 @@ import java.util.Locale; import org.gcube.application.geoportal.model.CentroidRecord; import org.gcube.application.geoportal.model.DataParsingException; import org.gcube.application.geoportal.model.PostgisTable; +import org.gcube.application.geoportal.utils.Files; import lombok.extern.slf4j.Slf4j; @@ -57,7 +55,7 @@ public class PostgisDBManager { Connection conn=null; - File sqlFolder=getFileFromResources("sql"); + File sqlFolder=Files.getFileFromResources("sql"); if(!sqlFolder.isDirectory()) throw new RuntimeException(sqlFolder.getAbsolutePath()+" is not a folder."); conn=DriverManager.getConnection(url,user,pwd); @@ -66,7 +64,7 @@ public class PostgisDBManager { Statement stmt=conn.createStatement(); for(File f:sqlFolder.listFiles()) { log.info("Executing "+f.getName()); - String sql=readFile(f.getAbsolutePath(),Charset.forName("UTF-8")); + String sql=Files.readFileAsString(f.getAbsolutePath(),Charset.forName("UTF-8")); stmt.execute(sql); } @@ -77,26 +75,10 @@ public class PostgisDBManager { } - private static String readFile(String path, Charset encoding) - throws IOException - { - byte[] encoded = Files.readAllBytes(Paths.get(path)); - return new String(encoded, encoding); - } - private static File getFileFromResources(String fileName) { - - ClassLoader classLoader =PostgisDBManager.class.getClassLoader(); - - URL resource = classLoader.getResource(fileName); - if (resource == null) { - throw new IllegalArgumentException("file is not found!"); - } else { - return new File(resource.getFile()); - } - - } + + private Connection conn=null; @@ -140,16 +122,21 @@ public class PostgisDBManager { } public PostgisTable.BBOX evaluateBoundingBox(PostgisTable table) throws SQLException, DataParsingException { - ResultSet rs=conn.createStatement().executeQuery("Select ST_Extent("+table.getGeometryColumnName()+") as extent from "+table.getTablename()); - rs.next(); - return PostgisTable.BBOX.parseST_Extent(rs.getString(1)); + ResultSet rs=conn.createStatement().executeQuery("Select ST_AsText(ST_Extent("+table.getGeometryColumnName()+")) as extent from "+table.getTablename()); + if(rs.next()) + return PostgisTable.BBOX.parseST_Extent(rs.getString("extent")); + else throw new SQLException("No extent returned"); } public PreparedStatement prepareInsertStatement(PostgisTable target, boolean createTable) throws SQLException { if(createTable) { - conn.createStatement().executeUpdate(target.getCreateStatement()); + String createStmt=target.getCreateStatement(); + log.debug("Executing create : "+createStmt); + conn.createStatement().executeUpdate(createStmt); } - return conn.prepareStatement(target.getInsertionStatement()); + String insertStmt=target.getInsertionStatement(); + log.debug("Preparing insert statement : "+insertStmt); + return conn.prepareStatement(insertStmt); } public void deleteTable(String tableName) throws SQLException { diff --git a/src/main/java/org/gcube/application/geoportal/model/PostgisTable.java b/src/main/java/org/gcube/application/geoportal/model/PostgisTable.java index da74d20..dd0c7dd 100644 --- a/src/main/java/org/gcube/application/geoportal/model/PostgisTable.java +++ b/src/main/java/org/gcube/application/geoportal/model/PostgisTable.java @@ -16,7 +16,9 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +@Slf4j @RequiredArgsConstructor @Getter public class PostgisTable { @@ -25,11 +27,13 @@ public class PostgisTable { @Getter @ToString public static class BBOX{ - + private static Pattern pattern = Pattern.compile("^(-?)(0|([1-9][0-9]*))(\\.[0-9]+)?$"); - + public static BBOX parseST_Extent(String extent) throws DataParsingException { //BOX(11.9122574810083 44.2514144864263,11.9761128271586 44.2912342569845) + log.debug("Parsing BBOX "+extent); + Matcher m=pattern.matcher(extent); if(m.groupCount()!=4) throw new DataParsingException("Invalid extent format "+extent); Double minLong=Double.parseDouble(m.group(0)); @@ -38,7 +42,7 @@ public class PostgisTable { Double maxLat=Double.parseDouble(m.group(3)); return new BBOX(maxLat, maxLong, minLat, minLong); } - + @NonNull private Double maxLat; @NonNull @@ -48,34 +52,33 @@ public class PostgisTable { @NonNull private Double minLong; } - - + + private static final NumberFormat DECIMAL_FORMAT=NumberFormat.getInstance(Locale.US); - - + + private static final String GEOMETRY_COLUMN_NAME="geom"; public static final PostgisTable fromCSVHeaders(List csvHeaders, GeometryType expectedType) { - String tableName=UUID.randomUUID().toString().toLowerCase(); + String tableName="l"+UUID.randomUUID().toString().toLowerCase().replaceAll("-", "_"); ArrayList fields=new ArrayList(); for(String csvHeader:csvHeaders) fields.add(sanitizeFieldName(csvHeader)); - String geometryColumnName=null; + String geometryColumnName=GEOMETRY_COLUMN_NAME; GeometryType geometryColumnType=expectedType; + fields.add(GEOMETRY_COLUMN_NAME); + switch(expectedType) { - case POINT :{ - fields.add("point"); - geometryColumnName="point"; + case POINT :{ if(!fields.contains("xcoord")) throw new RuntimeException("No xcoord in headers"); if(!fields.contains("ycoord")) throw new RuntimeException("No ycoord in headers"); break; } default : { - geometryColumnName="wkt"; - if(!fields.contains(geometryColumnName)) throw new RuntimeException("No "+geometryColumnName+" in headers"); + if(!fields.contains("wkt")) throw new RuntimeException("No wkt in headers"); break; } } @@ -95,22 +98,18 @@ public class PostgisTable { private String geometryColumnName; @NonNull private GeometryType geometryColumnType; - + @Setter private BBOX boundingBox=null; - - + + public String getCreateStatement() { StringBuilder stmt=new StringBuilder(); stmt.append("CREATE TABLE "+tablename+"( "); for(String field:fieldNames) { String fieldDefinition=null; - switch(field) { - case("point"):{ - fieldDefinition="geometry (POINT,4326)"; - break; - } - case("wkt"):{ + switch(field) { + case(GEOMETRY_COLUMN_NAME):{ switch(geometryColumnType) { case LINE:{ fieldDefinition="geometry (MULTILINESTRING,4326)"; @@ -120,7 +119,10 @@ public class PostgisTable { fieldDefinition="geometry (MULTIPOLYGON,4326)"; break; } - default : throw new RuntimeException("INCOHERENT DEFINITION"); + case POINT:{ + fieldDefinition="geometry (POINT,4326)"; + break; + } } break; } @@ -151,35 +153,35 @@ public class PostgisTable { public String getInsertionStatement() { StringBuilder fieldList=new StringBuilder(); StringBuilder fieldInsertion=new StringBuilder(); - + for(String field:fieldNames) { fieldList.append(field+","); - + if(field.equals(geometryColumnName)) { - fieldInsertion.append("ST_GeomFromText(?, 4326)"); + fieldInsertion.append("ST_GeomFromText(?, 4326),"); }else fieldInsertion.append("?,"); } fieldList.deleteCharAt(fieldList.lastIndexOf(",")); fieldInsertion.deleteCharAt(fieldInsertion.lastIndexOf(",")); - - - + + + return "Insert into "+tablename+" ("+fieldList+") VALUES ("+fieldInsertion+")"; } - + public void fillPreparedStatement(Map rowValues, PreparedStatement toFill) throws SQLException { int psFieldIndex=0; for(String field:fieldNames) { - + psFieldIndex++; - + switch(field) { case("point"):{ String xRepresentation=DECIMAL_FORMAT.format(Double.parseDouble(rowValues.get("xcoord"))); String yRepresentation=DECIMAL_FORMAT.format(Double.parseDouble(rowValues.get("ycoord"))); - + toFill.setString(psFieldIndex, "POINT("+xRepresentation+" "+ yRepresentation+")"); break; @@ -200,7 +202,7 @@ public class PostgisTable { } } } - + private static String sanitizeFieldName(String fieldName) { return fieldName.toLowerCase().replaceAll(" ", "_"); } diff --git a/src/test/java/org/gcube/application/geoportal/Centroids.java b/src/test/java/org/gcube/application/geoportal/Centroids.java index 067f058..425c4f7 100644 --- a/src/test/java/org/gcube/application/geoportal/Centroids.java +++ b/src/test/java/org/gcube/application/geoportal/Centroids.java @@ -1,11 +1,11 @@ package org.gcube.application.geoportal; import java.io.File; -import java.util.HashMap; import java.util.Map; import org.gcube.application.geoportal.model.CentroidRecord; import org.gcube.application.geoportal.model.DatabaseConnection; +import org.gcube.application.geoportal.utils.MainUtils; public class Centroids { @@ -26,7 +26,7 @@ public class Centroids { String cmd=args[0]; - Map options=asMap(args); + Map options=MainUtils.asMap(args); System.out.println("CMD : "+cmd); System.out.println("Options : "+options); @@ -35,7 +35,7 @@ public class Centroids { switch(cmd) { case "single":{ - PostgisDBManager.init(getMandatory("DBURL",options), getMandatory("DBUSER",options), getMandatory("DBPWD",options)); + PostgisDBManager.init(MainUtils.getMandatory("DBURL",options), MainUtils.getMandatory("DBUSER",options), MainUtils.getMandatory("DBPWD",options)); PostgisDBManager db=PostgisDBManager.get(); @@ -52,9 +52,9 @@ public class Centroids { case "csv":{ Uploader u=new Uploader(); - File csvFile=new File(getMandatory("csvFile", options)); + File csvFile=new File(MainUtils.getMandatory("csvFile", options)); - DatabaseConnection postgresDB=new DatabaseConnection(getMandatory("DBUSER",options), getMandatory("DBPWD",options),getMandatory("DBURL",options)); + DatabaseConnection postgresDB=new DatabaseConnection(MainUtils.getMandatory("DBUSER",options), MainUtils.getMandatory("DBPWD",options),MainUtils.getMandatory("DBURL",options)); u.insertCSVCentroids(csvFile, postgresDB, null); } @@ -65,23 +65,5 @@ public class Centroids { } - private static final String getMandatory(String key,Map map) { - if(map.containsKey(key)) return map.get(key); - else throw new RuntimeException("Missing mandatory parameter "+key); - } - private static final String getOptional(String key,Map map, String defaultValue) { - if(map.containsKey(key)) return map.get(key); - else return defaultValue; - } - - private static Map asMap(String[] args){ - HashMap toReturn=new HashMap<>(); - for(String arg:args) { - if(arg.contains("=")) - toReturn.put(arg.substring(2, arg.indexOf("=")), // skipping first 2 chars "--XX=" - arg.substring(arg.indexOf("=")+1)); - } - return toReturn; - } } diff --git a/src/test/java/org/gcube/application/geoportal/db/PostgisTableTests.java b/src/test/java/org/gcube/application/geoportal/db/PostgisTableTests.java index 3160b85..77f1d09 100644 --- a/src/test/java/org/gcube/application/geoportal/db/PostgisTableTests.java +++ b/src/test/java/org/gcube/application/geoportal/db/PostgisTableTests.java @@ -1,9 +1,42 @@ package org.gcube.application.geoportal.db; +import java.io.File; + +import org.gcube.application.geoportal.model.PostgisTable.GeometryType; +import org.gcube.application.geoportal.utils.CSV; +import org.gcube.application.geoportal.utils.Files; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + public class PostgisTableTests { - public PostgisTableTests() { - // TODO Auto-generated constructor stub + private static File csvFolder=null; + + @BeforeClass + public static void init() { + csvFolder=Files.getFileFromResources("csv"); } + + + @Test + public void testParseCSV() { + for(String csv:csvFolder.list()) { + GeometryType type=GeometryType.POINT; + if(csv.contains("linee")) + type=GeometryType.LINE; + else if(csv.contains("poligoni")) + type=GeometryType.POLYGON; + try { + CSV.tableFromCSV(csvFolder.getAbsolutePath()+File.separator+csv, type); + }catch(Throwable t) { + if(!csv.contains("wrong")) + Assert.fail(t.getMessage()); + } + + } + } + + }