From 1c0504a3da7b79aec379d8792183ac563516fad5 Mon Sep 17 00:00:00 2001 From: Francesco Mangiacrapa Date: Thu, 16 Jun 2016 13:03:03 +0000 Subject: [PATCH] 4250: Geonetwork Resolver upgrade: it must return only "private" Metadata Ids for CKAN harversting Task-Url: https://support.d4science.org/issues/4250 Added code to filter a list of (public) IDs from CSW GetRecordsResponse git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-transfer/uri-resolver@129136 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 10 +- GetResponseRecords.xml | 230 ++++++++++++++++++ distro/changelog.xml | 4 +- .../resolver/GeonetworkRequestCriteria.java | 78 ++++++ .../resolver/GeonetworkRequestDecoder.java | 100 ++++++++ .../resolver/UriResolverRewriteFilter.java | 19 +- .../gis/geonetwork/FilterGetRecords.java | 78 ++++++ .../gis/geonetwork/GeonetworkResolver.java | 38 ++- .../gis/util/GetResponseRecordFilter.java | 110 +++++++++ .../resolver/gis/util/NamespaceCsw.java | 90 +++++++ .../resolver/gis/util/NamespaceISO19139.java | 194 +++++++++++++++ 11 files changed, 929 insertions(+), 22 deletions(-) create mode 100644 GetResponseRecords.xml create mode 100644 src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestCriteria.java create mode 100644 src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestDecoder.java create mode 100644 src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java create mode 100644 src/main/java/org/gcube/datatransfer/resolver/gis/util/GetResponseRecordFilter.java create mode 100644 src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceCsw.java create mode 100644 src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceISO19139.java diff --git a/.classpath b/.classpath index 8da6250..6727def 100644 --- a/.classpath +++ b/.classpath @@ -1,5 +1,10 @@ + + + + + @@ -28,10 +33,5 @@ - - - - - diff --git a/GetResponseRecords.xml b/GetResponseRecords.xml new file mode 100644 index 0000000..41e625e --- /dev/null +++ b/GetResponseRecords.xml @@ -0,0 +1,230 @@ + + + + + + fao-species-map-heg + FAO aquatic species distribution map of Heterodontus galeatus + dataset + FAO + FIGIS + fishery + fisheries + aquatic species distribution + Species distribution + 276695 + Heterodontus galeatus + HEG + Crested bullhead shark + Heterodontus galeatus + Heterodontidae + HETERODONTIFORMES + 12656 + fao-species-map-heg + http://www.fao.org/figis/lod/flod/entities/codedentity/eb7d98c4c597b05ce18ce35345af2eedbbc50ef2 + biota + boundaries + 2015-10-07 + The main sources of information for the species distribution are the habitat description and geographic range contained in the published FAO Catalogues of Species (more details at http://www.fao.org/fishery/fishfinder ). Terms used in the descriptive context of the FAO Catalogues were converted in standard depth, geographic and ecological regions and inserted into a Geographic Information System. + + + fao-species-map-hef + FAO aquatic species distribution map of Heterodontus francisci + dataset + FAO + FIGIS + fishery + fisheries + aquatic species distribution + Species distribution + 276694 + Heterodontus francisci + HEF + Horn shark + Heterodontus francisci + Heterodontidae + HETERODONTIFORMES + 12655 + fao-species-map-hef + http://www.fao.org/figis/lod/flod/entities/codedentity/a2a9b6439ce74eaf2a28afc6bef57eec6ec950b0 + biota + boundaries + 2015-10-07 + The main sources of information for the species distribution are the habitat description and geographic range contained in the published FAO Catalogues of Species (more details at http://www.fao.org/fishery/fishfinder ). Terms used in the descriptive context of the FAO Catalogues were converted in standard depth, geographic and ecological regions and inserted into a Geographic Information System. + + + fao-fsa-map-41.1.3 + Salvador (Division 41.1.3 of FAO Major Area 41) + dataset + FAO + FIGIS + fishery + fisheries + fishery statistical areas + Area management/restriction/regulation zones and reporting units + http://www.fao.org/figis/lod/flod/entities/codedentity/415720c1e1813e22dd87f453e20b1fbd4c560207 + fao-fsa-map-41.1.3 + 41.1.3 + Salvador (Division 41.1.3 of FAO Major Area 41) + DIVISION + boundaries + fao-fsa-map-41.1 + 2015-10-13 + The FAO major fishing areas for statistical purpose are defined by the CWP handbook of fishery statistical standards available at http://www.fao.org/fishery/cwp/handbook/h/en + + + fao-species-map-cre + FAO aquatic species distribution map of Cancer pagurus + dataset + FAO + FIGIS + fishery + fisheries + aquatic species distribution + Species distribution + 2627 + http://www.fao.org/figis/lod/flod/entities/codedentity/c3518fda2e3c6765711757a92d24e57408eea429 + CRE + Edible crab + Cancer pagurus + Cancridae + BRACHYURA + fao-species-map-cre + biota + boundaries + 2015-10-07 + The main sources of information for the species distribution are the habitat description and geographic range contained in the published FAO Catalogues of Species (more details at http://www.fao.org/fishery/fishfinder ). Terms used in the descriptive context of the FAO Catalogues were converted in standard depth, geographic and ecological regions and inserted into a Geographic Information System. + + + fao-fsa-map-41.1.4 + Nothern Oceanic (Division 41.1.4 of FAO Major Area 41) + dataset + FAO + FIGIS + fishery + fisheries + fishery statistical areas + Area management/restriction/regulation zones and reporting units + http://www.fao.org/figis/lod/flod/entities/codedentity/b1d33b4be19ed2dc28e9b53b7087b77ac9821662 + fao-fsa-map-41.1.4 + 41.1.4 + Nothern Oceanic (Division 41.1.4 of FAO Major Area 41) + DIVISION + boundaries + fao-fsa-map-41.1 + 2015-10-13 + The FAO major fishing areas for statistical purpose are defined by the CWP handbook of fishery statistical standards available at http://www.fao.org/fishery/cwp/handbook/h/en + + + fao-species-map-hea + FAO aquatic species distribution map of Heterodontus ramalheira + dataset + FAO + FIGIS + fishery + fisheries + aquatic species distribution + Species distribution + 217626 + Heterodontus ramalheira + HEA + Whitespotted bullhead shark + Heterodontus ramalheira + Heterodontidae + HETERODONTIFORMES + 12661 + fao-species-map-hea + http://www.fao.org/figis/lod/flod/entities/codedentity/be06aeb63705419c7b4c0912078337d03dbe708d + biota + boundaries + 2015-10-07 + The main sources of information for the species distribution are the habitat description and geographic range contained in the published FAO Catalogues of Species (more details at http://www.fao.org/fishery/fishfinder ). Terms used in the descriptive context of the FAO Catalogues were converted in standard depth, geographic and ecological regions and inserted into a Geographic Information System. + + + fao-species-map-crb + FAO aquatic species distribution map of Callinectes sapidus + dataset + FAO + FIGIS + fishery + fisheries + aquatic species distribution + Species distribution + 2632 + http://www.fao.org/figis/lod/flod/entities/codedentity/3fe12b2c0394298dd5338da522bebadadf422326 + CRB + Blue crab + Callinectes sapidus + Portunidae + BRACHYURA + fao-species-map-crb + biota + boundaries + 2015-10-07 + The main sources of information for the species distribution are the habitat description and geographic range contained in the published FAO Catalogues of Species (more details at http://www.fao.org/fishery/fishfinder ). Terms used in the descriptive context of the FAO Catalogues were converted in standard depth, geographic and ecological regions and inserted into a Geographic Information System. + + + fao-species-map-zbl + FAO aquatic species distribution map of Bathothauma lyromma + dataset + FAO + FIGIS + fishery + fisheries + aquatic species distribution + Species distribution + 20437 + http://www.fao.org/figis/lod/flod/entities/codedentity/b9e91f69580e800b8d1f7d7ee232e7f0bdbd4b47 + ZBL + Lyre cranch squid + Bathothauma lyromma + Cranchiidae + CEPHALOPODA + fao-species-map-zbl + biota + boundaries + 2015-10-27 + The main sources of information for the species distribution are the habitat description and geographic range contained in the published FAO Catalogues of Species (more details at http://www.fao.org/fishery/fishfinder ). Terms used in the descriptive context of the FAO Catalogues were converted in standard depth, geographic and ecological regions and inserted into a Geographic Information System. + + + fao-fsa-map-41.1.1 + Amazon (Division 41.1.1 of FAO Major Area 41) + dataset + FAO + FIGIS + fishery + fisheries + fishery statistical areas + Area management/restriction/regulation zones and reporting units + http://www.fao.org/figis/lod/flod/entities/codedentity/323ea512949786623dccbcd8320f24a59686a485 + fao-fsa-map-41.1.1 + 41.1.1 + Amazon (Division 41.1.1 of FAO Major Area 41) + DIVISION + boundaries + fao-fsa-map-41.1 + 2015-10-13 + The FAO major fishing areas for statistical purpose are defined by the CWP handbook of fishery statistical standards available at http://www.fao.org/fishery/cwp/handbook/h/en + + + fao-fsa-map-41.1.2 + Natal (Division 41.1.2 of FAO Major Area 41) + dataset + FAO + FIGIS + fishery + fisheries + fishery statistical areas + Area management/restriction/regulation zones and reporting units + http://www.fao.org/figis/lod/flod/entities/codedentity/ec79700361176dbf80815a9235d71bde3fbc0542 + fao-fsa-map-41.1.2 + 41.1.2 + Natal (Division 41.1.2 of FAO Major Area 41) + DIVISION + boundaries + fao-fsa-map-41.1 + 2015-10-13 + The FAO major fishing areas for statistical purpose are defined by the CWP handbook of fishery statistical standards available at http://www.fao.org/fishery/cwp/handbook/h/en + + + \ No newline at end of file diff --git a/distro/changelog.xml b/distro/changelog.xml index 4fc7bd6..a826ea8 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -44,7 +44,9 @@ - [Feature #4207] ri Resolver upgrade: it must support new Geonetwork Manager + [Feature #4207] Uri Resolver upgrade: it must support new Geonetwork Manager + + [Task #4250] Geonetwork Resolver upgrade: it must return only "private" Metadata Ids for CKAN harversting \ No newline at end of file diff --git a/src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestCriteria.java b/src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestCriteria.java new file mode 100644 index 0000000..52e3acc --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestCriteria.java @@ -0,0 +1,78 @@ +/** + * + */ +package org.gcube.datatransfer.resolver; + + +/** + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Jun 15, 2016 + */ +public class GeonetworkRequestCriteria { + + private String scope; + private final String FILTER_PUBLIC_IDS = "FILTER_PUBLIC_IDS"; + private boolean valueOfFilterPublicIds; + + GeonetworkRequestCriteria(String scope, boolean valueOfFilterPublicIds){ + this.scope = scope; + this.valueOfFilterPublicIds = valueOfFilterPublicIds; + } + + + /** + * @return the scope + */ + public String getScope() { + + return scope; + } + + + /** + * @return the valueOfFilterPublicIds + */ + public boolean isValueOfFilterPublicIds() { + + return valueOfFilterPublicIds; + } + + + /** + * @param scope the scope to set + */ + public void setScope(String scope) { + + this.scope = scope; + } + + + /** + * @param valueOfFilterPublicIds the valueOfFilterPublicIds to set + */ + public void setValueOfFilterPublicIds(boolean valueOfFilterPublicIds) { + + this.valueOfFilterPublicIds = valueOfFilterPublicIds; + } + + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + + StringBuilder builder = new StringBuilder(); + builder.append("GeonetworkQueryCriteria [scope="); + builder.append(scope); + builder.append(", FILTER_PUBLIC_IDS="); + builder.append(FILTER_PUBLIC_IDS); + builder.append(", valueOfFilterPublicIds="); + builder.append(valueOfFilterPublicIds); + builder.append("]"); + return builder.toString(); + } + + +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestDecoder.java b/src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestDecoder.java new file mode 100644 index 0000000..dabf444 --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/GeonetworkRequestDecoder.java @@ -0,0 +1,100 @@ +/** + * + */ +package org.gcube.datatransfer.resolver; + +import javax.servlet.ServletException; + +import org.apache.commons.lang.StringUtils; +import org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * The Class GeonetworkScopeDecoder. + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Gets the new uri to forward the request to {@link GeonetworkResolver} + * Jun 15, 2016 + */ +public class GeonetworkRequestDecoder { + + public static final Logger logger = LoggerFactory.getLogger(GeonetworkRequestDecoder.class); + + private String newURI; + + /** + * Instantiates a new geonetwork request decoder. + * + * @param theServletPath the the servlet path + * @param queryString the query string + * @throws ServletException the servlet exception + */ + public GeonetworkRequestDecoder(String theServletPath, String queryString) throws ServletException{ + + logger.debug("is geonetwork request"); + String path = theServletPath; + String pathWithoutGN = path.substring(UriResolverRewriteFilter.SERVLET_GEONETWORK.length()+1, path.length()); + logger.debug("servlet path without "+UriResolverRewriteFilter.SERVLET_GEONETWORK + " is: " +pathWithoutGN); + String[] params = pathWithoutGN.split("/"); + if(params[0]==null || params[0].isEmpty()){ + logger.error("Scope is null or empty, you must set a valid scope /geonetwork/root_vo_vre"); + throw new ServletException("Scope is null or empty, you must set a valid scope /geonetwork/root_vo_vre"); + } + + GeonetworkRequestCriteria rc = getGeonetworkRequestCriteria(params[0]); + logger.debug("scope value is: "+rc.getScope()); + newURI = UriResolverRewriteFilter.SERVLET_GEONETWORK + "?" + GeonetworkResolver.SCOPE + "=" + rc.getScope() +"&"+ GeonetworkResolver.PARAMETER_FILTER_PUBLIC_IDS +"="+rc.isValueOfFilterPublicIds(); + logger.debug(GeonetworkResolver.PARAMETER_FILTER_PUBLIC_IDS +" is: "+rc.isValueOfFilterPublicIds()); + + if(params.length>1){ + String remainPath = ""; +// newURI +="&remainPath="; + for (int i = 1; i < params.length; i++) { + String httpGetParam = params[i]; + if(httpGetParam!=null && !httpGetParam.isEmpty()) + remainPath+="/"+httpGetParam; + } + newURI +="&"+GeonetworkResolver.REMAIN_PATH+"="+remainPath; + } + + if(queryString!=null && !queryString.isEmpty()) + newURI+="&"+queryString; + } + + + + /** + * Gets the geonetwork request criteria. + * Parses a request like root_vo_vre#filterPublicIds or root_vo_vre + * + * @param request the request + * @return the geonetwork request criteria + */ + private static GeonetworkRequestCriteria getGeonetworkRequestCriteria(String request){ + logger.debug("Read request: "+request); + int index = request.indexOf(UriResolverRewriteFilter.REQUEST_PARAMETER_SEPARATOR); + String scope = ""; + boolean filterPublicIds = false; + if(index!=-1){ + scope = request.substring(0,index); + filterPublicIds = StringUtils.containsIgnoreCase(UriResolverRewriteFilter.PARAMETER_FILTER_PUBLIC_IDS, request); + }else + scope = request; + + return new GeonetworkRequestCriteria("/"+scope.replaceAll("_", "/"), filterPublicIds); + } + + + /** + * Gets the new uri. + * + * @return the newURI + */ + public String getNewURI() { + + return newURI; + } + +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/UriResolverRewriteFilter.java b/src/main/java/org/gcube/datatransfer/resolver/UriResolverRewriteFilter.java index afdcb4e..48c87ef 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/UriResolverRewriteFilter.java +++ b/src/main/java/org/gcube/datatransfer/resolver/UriResolverRewriteFilter.java @@ -13,7 +13,6 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; -import org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +26,8 @@ import org.slf4j.LoggerFactory; public class UriResolverRewriteFilter implements Filter{ public static final String SERVLET_GEONETWORK = "/geonetwork"; + public static final String PARAMETER_FILTER_PUBLIC_IDS = "filterpublicids"; + public static final String REQUEST_PARAMETER_SEPARATOR = "#"; protected static final String SMP_ID = "smp-id"; protected static final String SERVLET_RESOLVER_BY_ID = "id"; protected static final Logger logger = LoggerFactory.getLogger(UriResolverRewriteFilter.class); @@ -63,8 +64,7 @@ public class UriResolverRewriteFilter implements Filter{ //IS A REQUEST FOR GEONETWORK AUTHENTICATION? (CKAN HARVESTING?) if(isGeonetworkRequest(multiReadRequest.getServletPath())){ - logger.debug("is geonetwork"); - String path = multiReadRequest.getServletPath(); + /*String path = multiReadRequest.getServletPath(); String pathWithoutGN = path.substring(SERVLET_GEONETWORK.length()+1, path.length()); logger.debug("servlet path without "+SERVLET_GEONETWORK + " is: " +pathWithoutGN); String[] params = pathWithoutGN.split("/"); @@ -93,11 +93,14 @@ public class UriResolverRewriteFilter implements Filter{ logger.debug("forward "+newURI); //BODY DEBUG - /* - String readBody = IOUtils.toString(multiReadRequest.getReader()); - logger.debug("Read body request: "+readBody); - */ - multiReadRequest.getRequestDispatcher(newURI).forward(multiReadRequest, response); +// String readBody = IOUtils.toString(multiReadRequest.getReader()); +// logger.debug("Read body request: "+readBody); + multiReadRequest.getRequestDispatcher(newURI).forward(multiReadRequest, response);*/ + logger.debug("is geonetwork request"); + GeonetworkRequestDecoder grd = new GeonetworkRequestDecoder(multiReadRequest.getServletPath(), queryString); + logger.debug("forward to: "+grd.getNewURI()); + multiReadRequest.getRequestDispatcher(grd.getNewURI()).forward(multiReadRequest, response); + }else{ //IS WORKSPACE REQUEST? if (queryString == null) { // IS A /XXXXX diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java b/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java new file mode 100644 index 0000000..ef38782 --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/FilterGetRecords.java @@ -0,0 +1,78 @@ +/** + * + */ +package org.gcube.datatransfer.resolver.gis.geonetwork; + +import it.geosolutions.geonetwork.util.GNSearchRequest; +import it.geosolutions.geonetwork.util.GNSearchResponse; +import it.geosolutions.geonetwork.util.GNSearchResponse.GNMetadata; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.gcube.spatial.data.geonetwork.GeoNetwork; +import org.gcube.spatial.data.geonetwork.GeoNetworkReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * The Class FilterGetRecords. + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Jun 15, 2016 + */ +public class FilterGetRecords { + + /** + * + */ + private static final String CSW_GET_RECORDS = "csw:GetRecords"; + + private static final Logger logger = LoggerFactory.getLogger(FilterGetRecords.class); + + private List foundPublicIds = null; + + + /** + * Instantiates a new filter get records. + * + * @param readBody the read body + */ + public FilterGetRecords(String readBody) { + + if(readBody!=null && !readBody.isEmpty() && readBody.contains(CSW_GET_RECORDS)){ + logger.info("Is "+CSW_GET_RECORDS+" request, getting public ids"); + GeoNetworkReader reader; + try { + reader = GeoNetwork.get(); + final GNSearchRequest req=new GNSearchRequest(); + req.addParam(GNSearchRequest.Param.any,""); + GNSearchResponse resp=reader.query(req); + + foundPublicIds = new ArrayList(); + Iterator iterator=resp.iterator(); + while(iterator.hasNext()){ + foundPublicIds.add(iterator.next().getUUID()); + } + logger.info("Public Metadata ids are: "+foundPublicIds.size()); + }catch (Exception e) { + logger.error("Error during sending GNSearchRequest: ",e); + } + } + } + + + /** + * Gets the found public ids. + * + * @return the foundPublicIds + */ + public List getFoundPublicIds() { + + return foundPublicIds==null || foundPublicIds.isEmpty()? null: foundPublicIds; + } + + +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/GeonetworkResolver.java b/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/GeonetworkResolver.java index 22be5cb..e9b95e2 100644 --- a/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/GeonetworkResolver.java +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/geonetwork/GeonetworkResolver.java @@ -27,6 +27,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.datatransfer.resolver.UriResolverRewriteFilter; import org.gcube.datatransfer.resolver.gis.GeoRuntimeReader; import org.gcube.datatransfer.resolver.gis.GeoRuntimeReader.GEO_SERVICE; import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter; @@ -34,6 +35,7 @@ import org.gcube.datatransfer.resolver.gis.GeonetworkInstance; import org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface; import org.gcube.datatransfer.resolver.gis.entity.ServerParameters; import org.gcube.datatransfer.resolver.gis.exception.IllegalArgumentException; +import org.gcube.datatransfer.resolver.gis.util.GetResponseRecordFilter; import org.gcube.spatial.data.geonetwork.configuration.Configuration; import org.gcube.spatial.data.geonetwork.model.Account; import org.gcube.spatial.data.geonetwork.model.Account.Type; @@ -74,6 +76,7 @@ public class GeonetworkResolver extends HttpServlet{ public static final String RESET_CACHE = "resetcache"; public static final String RESET_CACHED_SCOPE = "resetcachedscope"; public static final String CSW_SERVER = "srv/en/csw"; + public static final String PARAMETER_FILTER_PUBLIC_IDS = UriResolverRewriteFilter.PARAMETER_FILTER_PUBLIC_IDS; /** The logger. */ private static final Logger logger = LoggerFactory.getLogger(GeonetworkResolver.class); @@ -240,6 +243,8 @@ public class GeonetworkResolver extends HttpServlet{ logger.info("doPost running..."); String scope = req.getParameter(SCOPE); String remainValue = req.getParameter(REMAIN_PATH); + String filterPublicIDs = req.getParameter(PARAMETER_FILTER_PUBLIC_IDS); + boolean filterPublicMetadataIDs = false; if (scope == null || scope.equals("")) { logger.debug("Scope not found"); @@ -247,7 +252,12 @@ public class GeonetworkResolver extends HttpServlet{ return; } + if(filterPublicIDs!=null && Boolean.parseBoolean(filterPublicIDs)){ + filterPublicMetadataIDs = true; + } + logger.info("SCOPE is: " + scope); + logger.info(PARAMETER_FILTER_PUBLIC_IDS +" is "+filterPublicMetadataIDs); try { @@ -275,12 +285,6 @@ public class GeonetworkResolver extends HttpServlet{ // logger.debug("doPost read body request: "+readBody); ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); - if(account.getUser()!=null){ - boolean authorized = GNAuthentication.login(httpUtils, geonetworkParams.getUrl(), account.getUser(), account. getPassword()); - logger.trace("Authorized on "+geonetworkParams +" ? "+authorized); - }else - logger.info("Skipping authentication ckan user is null"); - // SPECIFIC HANDLER FOR GEONETWORK REQUEST: /srv/en/mef.export String gnCSWlURL; if(remainValue!=null && remainValue.compareTo(SRV_EN_MEF_EXPORT)==0){ @@ -293,7 +297,7 @@ public class GeonetworkResolver extends HttpServlet{ data = UUID+"="+uuid; } if(data!=null){ - logger.debug("Writing "+data +" to byte array"); + logger.debug("Writing "+data +" into byte array"); byteArray.write(data.getBytes()); }else IOUtils.copy(req.getReader(), byteArray); @@ -304,10 +308,22 @@ public class GeonetworkResolver extends HttpServlet{ IOUtils.copy(req.getReader(), byteArray); } + FilterGetRecords filterGetRecords = null; + if(filterPublicMetadataIDs){ + filterGetRecords = new FilterGetRecords(byteArray.toString()); + } + + if(account.getUser()!=null){ + boolean authorized = GNAuthentication.login(httpUtils, geonetworkParams.getUrl(), account.getUser(), account. getPassword()); + logger.trace("Authorized on "+geonetworkParams +" ? "+authorized); + }else + logger.info("Skipping authentication ckan user is null"); + + logger.info("Sending CSW POST request to URL: "+gnCSWlURL); logger.info("Content-Type: "+req.getContentType()); //DEBUG - logger.debug("POST - BODY : "+byteArray.toString()); + logger.trace("POST - BODY : "+byteArray.toString()); InputStream in = httpUtils.post(gnCSWlURL, new ByteArrayInputStream(byteArray.toByteArray()), req.getContentType(), req.getParameterMap()); //END DEBUG logger.info("Response return Content-Type: "+httpUtils.getLastContentType()); @@ -320,6 +336,12 @@ public class GeonetworkResolver extends HttpServlet{ return; } try{ + + if(filterGetRecords!=null){ + logger.info("I'm removing list of public IDs, Is it right? "+filterGetRecords); + in = GetResponseRecordFilter.removeSummaryIdsByListIds(new ByteArrayInputStream(byteArray.toByteArray()), filterGetRecords.getFoundPublicIds()); + } + int bytes = IOUtils.copy(in, out); if(bytes==0) logger.warn("ResponseBody is empty, returning empty resp"); diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/util/GetResponseRecordFilter.java b/src/main/java/org/gcube/datatransfer/resolver/gis/util/GetResponseRecordFilter.java new file mode 100644 index 0000000..c6483c0 --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/util/GetResponseRecordFilter.java @@ -0,0 +1,110 @@ +/** + * + */ + +package org.gcube.datatransfer.resolver.gis.util; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * The Class GetResponseRecordFilter. + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it Jun 16, 2016 + */ +public class GetResponseRecordFilter { + + public static Logger logger = LoggerFactory.getLogger(GetResponseRecordFilter.class); + + /** + * Delete summary record. + * + * @param doc + * the doc + * @param identifier + * the identifier + */ + private static void deleteSummaryRecord(Document doc, String identifier) { + + // list + NodeList nodes = doc.getElementsByTagName("csw:SummaryRecord"); + logger.trace("SummaryRecord are: " + nodes.getLength()); + for (int i = 0; i < nodes.getLength(); i++) { + Element summaryRecord = (Element) nodes.item(i); + // + Element id = (Element) summaryRecord.getElementsByTagName("dc:identifier").item(0); + String idValue = id.getTextContent(); + logger.trace("Summary dc:identifier is: " + idValue); + if (idValue.equals(identifier)) { + summaryRecord.getParentNode().removeChild(summaryRecord); + logger.trace("Removed child " + idValue); + } + } + } + + + /** + * Removes the summary ids by list ids. + * + * @param getRecordsResponse the get records response + * @param idsToRemove the ids to remove + * @return the input stream + * @throws IOException Signals that an I/O exception has occurred. + */ + public static InputStream removeSummaryIdsByListIds(InputStream getRecordsResponse, List idsToRemove) throws IOException { + + try { + // logger.trace("getRecordsResponse is: "+IOUtils.toString(getRecordsResponse)); + BufferedInputStream bis = new BufferedInputStream(getRecordsResponse); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setValidating(false); + DocumentBuilder db = dbf.newDocumentBuilder(); + Document doc = db.parse(bis); + for (String identifier : idsToRemove) { + deleteSummaryRecord(doc, identifier); + } + return documentToInputStream(doc); + } + catch (Exception e) { + logger.error("An error occurred during removing IDS by List: ", e); + return getRecordsResponse; + } + } + + + /** + * Document to input stream. + * + * @param xml the xml + * @return the input stream + * @throws Exception the exception + */ + private static final InputStream documentToInputStream(Document xml) throws Exception { + Transformer tf = TransformerFactory.newInstance().newTransformer(); + tf.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + tf.setOutputProperty(OutputKeys.INDENT, "yes"); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + StreamResult outputTarget = new StreamResult(outputStream); + tf.transform(new DOMSource(xml), outputTarget); + return new ByteArrayInputStream(outputStream.toByteArray()); +// return out.toString(); + } +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceCsw.java b/src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceCsw.java new file mode 100644 index 0000000..418a95c --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceCsw.java @@ -0,0 +1,90 @@ +/** + * + */ +package org.gcube.datatransfer.resolver.gis.util; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import javax.xml.XMLConstants; +import javax.xml.namespace.NamespaceContext; + + +/** + * The Class NamespaceCsw. + * + * @author Francesco Mangiacrapa francesco.mangiacrapa@isti.cnr.it + * Jun 16, 2016 + */ +public class NamespaceCsw extends NamespaceISO19139 implements NamespaceContext { + + private Map mapPrefix; + + + /** + * Instantiates a new namespace csw. + */ + public NamespaceCsw() { + mapPrefix = new HashMap(); + mapPrefix.put(getPrefixGMD(), getNamespaceGMD()); + mapPrefix.put(getPrefixCSW(), getNamespaceCSW()); + mapPrefix.put(getPrefixOWS(), getNamespaceOWS()); + mapPrefix.put(getPrefixDC(), getNamespaceDC()); + mapPrefix.put(getPrefixGCO(), getNamespaceGCO()); + mapPrefix.put(getPrefixXLINK(), getNamespaceXLINK()); + mapPrefix.put(getPrefixSRV(), getNamespaceSRV()); + mapPrefix.put(getPrefixXSI(), getNamespaceXSI()); + mapPrefix.put(getPrefixGML(), getNamespaceGML()); + mapPrefix.put(getPrefixGTS(), getNamespaceGTS()); + mapPrefix.put(getPrefixGEONET(), getNamespaceGEONET()); + mapPrefix.put(getPrefixDCT(), getNamespaceDCT()); + }; + + /** + * Returns an iterator to every prefix in this namespace. + * + * @return the prefix iterator + */ + public Iterator getPrefixIterator() { + return mapPrefix.keySet().iterator(); + } + + + /* (non-Javadoc) + * @see javax.xml.namespace.NamespaceContext#getNamespaceURI(java.lang.String) + */ + public String getNamespaceURI(String prefix) { + if (prefix == null) + throw new IllegalArgumentException("No prefix provided!"); + + if (mapPrefix.containsKey(prefix)) + return mapPrefix.get(prefix); + else + return XMLConstants.NULL_NS_URI; + + } + + /* (non-Javadoc) + * @see javax.xml.namespace.NamespaceContext#getPrefix(java.lang.String) + */ + @Override + public String getPrefix(String namespaceURI) { + + for (String prefix : mapPrefix.keySet()) { + String nu = mapPrefix.get(prefix); + if(nu.compareTo(namespaceURI)==0) + return prefix; + } + return null; + } + + /* (non-Javadoc) + * @see javax.xml.namespace.NamespaceContext#getPrefixes(java.lang.String) + */ + @Override + public Iterator getPrefixes(String namespaceURI) { + + return mapPrefix.keySet().iterator(); + } +} diff --git a/src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceISO19139.java b/src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceISO19139.java new file mode 100644 index 0000000..7399ec9 --- /dev/null +++ b/src/main/java/org/gcube/datatransfer/resolver/gis/util/NamespaceISO19139.java @@ -0,0 +1,194 @@ +/** + * + */ +package org.gcube.datatransfer.resolver.gis.util; + + +public abstract class NamespaceISO19139 { + + + //Namespaces + private final String namespaceCSW = "http://www.opengis.net/cat/csw/2.0.2"; + private final String namespaceDC = "http://purl.org/dc/elements/1.1/"; + private final String namespaceOWS = "http://www.opengis.net/ows"; + private final String namespaceGMD = "http://www.isotc211.org/2005/gmd"; + private final String namespaceGCO = "http://www.isotc211.org/2005/gco"; + private final String namespaceXLINK = "http://www.w3.org/1999/xlink"; + private final String namespaceSRV = "http://www.isotc211.org/2005/srv"; + private final String namespaceXSI = "http://www.w3.org/2001/XMLSchema-instance"; + private final String namespaceGML = "http://www.opengis.net/gml"; + private final String namespaceGTS = "http://www.isotc211.org/2005/gts"; + private final String namespaceGEONET = "http://www.fao.org/geonetwork"; + private final String namespaceGMX = "http://www.isotc211.org/2005/gmx"; + private final String namespaceWMS = "http://www.opengis.net/wms"; + private final String namespaceDCT = "http://purl.org/dc/terms/"; + + //Prefixs + private final String prefixCSW = "csw"; + private final String prefixGMD = "gmd"; + private final String prefixOWS = "ows"; + private final String prefixDC = "dc"; + private final String prefixDCT = "dct"; + private final String prefixGCO = "gco"; + private final String prefixXLINK= "xlink"; + private final String prefixSRV = "srv"; + private final String prefixXSI = "xsi"; + private final String prefixGML = "gml"; + private final String prefixGTS = "gts"; + private final String prefixGEONET = "geonet"; + private final String prefixGMX = "gmx"; + private final String prefixWMS = "wms"; + + + public String getNamespaceCSW() { + return namespaceCSW; + } + public String getNamespaceDC() { + return namespaceDC; + } + public String getNamespaceOWS() { + return namespaceOWS; + } + public String getNamespaceGMD() { + return namespaceGMD; + } + public String getNamespaceGCO() { + return namespaceGCO; + } + public String getNamespaceXLINK() { + return namespaceXLINK; + } + public String getNamespaceSRV() { + return namespaceSRV; + } + public String getNamespaceXSI() { + return namespaceXSI; + } + public String getNamespaceGML() { + return namespaceGML; + } + public String getNamespaceGTS() { + return namespaceGTS; + } + public String getNamespaceGEONET() { + return namespaceGEONET; + } + public String getPrefixCSW() { + return prefixCSW; + } + public String getPrefixGMD() { + return prefixGMD; + } + public String getPrefixOWS() { + return prefixOWS; + } + public String getPrefixDC() { + return prefixDC; + } + public String getPrefixGCO() { + return prefixGCO; + } + public String getPrefixXLINK() { + return prefixXLINK; + } + public String getPrefixSRV() { + return prefixSRV; + } + public String getPrefixGML() { + return prefixGML; + } + public String getPrefixGTS() { + return prefixGTS; + } + public String getPrefixGEONET() { + return prefixGEONET; + } + public String getPrefixXSI() { + return prefixXSI; + } + public String getPrefixGMX() { + return prefixGMX; + } + public String getNamespaceGMX() { + return namespaceGMX; + } + public String getPrefixWMS() { + return prefixWMS; + } + public String getNamespaceWMS() { + return namespaceWMS; + } + public String getNamespaceDCT() { + return namespaceDCT; + } + public String getPrefixDCT() { + return prefixDCT; + } + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + + StringBuilder builder = new StringBuilder(); + builder.append("NamespaceISO19139 [namespaceCSW="); + builder.append(namespaceCSW); + builder.append(", namespaceDC="); + builder.append(namespaceDC); + builder.append(", namespaceOWS="); + builder.append(namespaceOWS); + builder.append(", namespaceGMD="); + builder.append(namespaceGMD); + builder.append(", namespaceGCO="); + builder.append(namespaceGCO); + builder.append(", namespaceXLINK="); + builder.append(namespaceXLINK); + builder.append(", namespaceSRV="); + builder.append(namespaceSRV); + builder.append(", namespaceXSI="); + builder.append(namespaceXSI); + builder.append(", namespaceGML="); + builder.append(namespaceGML); + builder.append(", namespaceGTS="); + builder.append(namespaceGTS); + builder.append(", namespaceGEONET="); + builder.append(namespaceGEONET); + builder.append(", namespaceGMX="); + builder.append(namespaceGMX); + builder.append(", namespaceWMS="); + builder.append(namespaceWMS); + builder.append(", namespaceDCT="); + builder.append(namespaceDCT); + builder.append(", prefixCSW="); + builder.append(prefixCSW); + builder.append(", prefixGMD="); + builder.append(prefixGMD); + builder.append(", prefixOWS="); + builder.append(prefixOWS); + builder.append(", prefixDC="); + builder.append(prefixDC); + builder.append(", prefixDCT="); + builder.append(prefixDCT); + builder.append(", prefixGCO="); + builder.append(prefixGCO); + builder.append(", prefixXLINK="); + builder.append(prefixXLINK); + builder.append(", prefixSRV="); + builder.append(prefixSRV); + builder.append(", prefixXSI="); + builder.append(prefixXSI); + builder.append(", prefixGML="); + builder.append(prefixGML); + builder.append(", prefixGTS="); + builder.append(prefixGTS); + builder.append(", prefixGEONET="); + builder.append(prefixGEONET); + builder.append(", prefixGMX="); + builder.append(prefixGMX); + builder.append(", prefixWMS="); + builder.append(prefixWMS); + builder.append("]"); + return builder.toString(); + } + +} \ No newline at end of file