package org.gcube.datatransfer.resolver.services; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.datatransfer.resolver.GeonetworkRequestFilterParameters; import org.gcube.datatransfer.resolver.GeonetworkRequestFilterParameters.MODE; import org.gcube.datatransfer.resolver.GeonetworkRequestFilterParameters.VISIBILITY; import org.gcube.datatransfer.resolver.SingleFileStreamingOutput; import org.gcube.datatransfer.resolver.Util; import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter; import org.gcube.datatransfer.resolver.gis.GeonetworkInstance; import org.gcube.datatransfer.resolver.gis.exception.GeonetworkInstanceException; import org.gcube.datatransfer.resolver.gis.geonetwork.HTTPCallsUtils; import org.gcube.datatransfer.resolver.gis.geonetwork.HTTPCallsUtils.HttpResponse; import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.datatransfer.resolver.services.exceptions.InternalServerException; import org.gcube.spatial.data.geonetwork.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The Class GeonetworkResolver. * * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) * Oct 23, 2018 */ @Path("geonetwork") public class GeonetworkResolver { private static Logger logger = LoggerFactory.getLogger(GeonetworkResolver.class); public static final String PATH_PARAM_MODE = "mode"; public static final String PATH_PARAM_REQUEST_DELIMITER = "requestDelimiter"; public static final String PATH_PARAM_OWNER = "owner"; private static final String PATH_PARAM_VISIBILITY = "visibility"; private static final String PATH_PARAM_SCOPE = "scope"; public static final String CSW_SERVER = "srv/en/csw"; public static final String VALUE_OF_REQUEST_DELIMITIER = "$$"; public static final String PATH_PARAM_REMAINING = "remainPath"; public static final String QUERY_PARAM_RESET_CACHE_PARAM = "resetcache"; public static final String QUERY_PARAM_RESET_CACHED_SCOPE_PARAM = "resetcachedscope"; public static final String SRV_EN_MEF_EXPORT = "/srv/en/mef.export"; //MEF Geonetwork service public static final String UUID = "uuid"; protected Map cacheGNInstances; private String help = "https://wiki.gcube-system.org/gcube/GCube_Resource_Catalogue#Geonetwork_Resolver"; /** * Gets the geonetwork request criteria. * Creates a request criteria from input parameter pathWithoutGN * The parameter pathWithoutGN should be an ordered string (like REST request): * SCOPE/MODE/VISIBILITY/OWNER/$$ * SCOPE must be: ROOT|VO|VRE * MODE must be: {@link MODE} * VISIBILITY must be: {@link VISIBILITY} * OWNER (is optional): filter by owner * * @param req the req * @param mode the mode * @param scope the scope * @param visibility the visibility * @param owner the owner * @param requestDelimiter the request delimiter * @param resetCache the reset cache * @param resetScope the reset scope * @return the geonetwork request criteria */ @GET @Path("{"+PATH_PARAM_SCOPE+"}/{"+PATH_PARAM_MODE+"}/{"+PATH_PARAM_VISIBILITY+"}/{"+PATH_PARAM_OWNER+"}/$${remainPath:(/[^?$]+)?}") public Response submitGet(@Context HttpServletRequest req, @PathParam(PATH_PARAM_SCOPE) @Nullable String scope, @PathParam(PATH_PARAM_MODE) @Nullable String mode, @PathParam(PATH_PARAM_VISIBILITY) @Nullable String visibility, @PathParam(PATH_PARAM_OWNER) @Nullable String owner, @PathParam("remainPath") @Nullable String remainPath, @QueryParam(QUERY_PARAM_RESET_CACHE_PARAM) @Nullable String resetCache, @QueryParam(QUERY_PARAM_RESET_CACHED_SCOPE_PARAM) @Nullable String resetScope) { logger.info(this.getClass().getSimpleName()+" GET starts..."); logger.info("Params are [mode: "+mode+", scope: "+scope+", visibility: "+visibility+", owner: "+owner+", remainPath: "+remainPath+"]"); if(scope==null || scope.isEmpty()){ logger.error("Path Parameter 'scope' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'scope'", GeonetworkResolver.class, help); } if(mode==null || mode.isEmpty()){ logger.error("Path Parameter 'scope' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'mode'", GeonetworkResolver.class, help); } mode = mode.toUpperCase(); try{ MODE.valueOf(mode); }catch(Exception e){ List toPrint = Arrays.asList(MODE.values()); logger.error("The 'mode' parameter is wrong, Have you pass a valid parameter MODE like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'mode' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } if(visibility==null){ logger.error("Path Parameter 'visibility' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'visibility'", GeonetworkResolver.class, help); } visibility = visibility.toUpperCase(); try{ VISIBILITY.valueOf(visibility); }catch (Exception e) { List toPrint = Arrays.asList(VISIBILITY.values()); logger.error("The 'visibility' parameter is wrong, Have you pass a valid parameter VISIBILITY like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'visibility' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } /*if(requestDelimiter==null || requestDelimiter.compareTo(VALUE_OF_REQUEST_DELIMITIER)!=0){ logger.error("Path Parameter to REQUEST_DELIMITIER '"+VALUE_OF_REQUEST_DELIMITIER+"' not found"); ExceptionManager.throwBadRequestException(req, "Path Parameter '"+VALUE_OF_REQUEST_DELIMITIER+"' not found as REQUEST DELIMITER", GeonetworkResolver.class, help); }*/ if(resetCache!=null && Boolean.parseBoolean(resetCache)){ purgeCacheGeonetworkInstances(); } if(resetScope!=null && Boolean.parseBoolean(resetScope)){ resetGeonetoworkInstanceCacheForScope(scope); } String fullURL = Util.getFullURL(req); int index = fullURL.indexOf(GeonetworkRequestFilterParameters.REQUEST_DELIMITIER); int delimiterIndex = index+GeonetworkRequestFilterParameters.REQUEST_DELIMITIER.length(); //BUILDING REMAINING PATH WITHOUT GeonetworkRequestFilterParameters.REQUEST_DELIMITIER if(delimiterIndex toPrint = Arrays.asList(MODE.values()); logger.error("The 'mode' parameter is wrong, Have you pass a valid parameter MODE like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'mode' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } if(visibility==null){ logger.error("Path Parameter 'visibility' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'visibility'", GeonetworkResolver.class, help); } visibility = visibility.toUpperCase(); try{ VISIBILITY.valueOf(visibility); }catch (Exception e) { List toPrint = Arrays.asList(VISIBILITY.values()); logger.error("The 'visibility' parameter is wrong, Have you pass a valid parameter VISIBILITY like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'visibility' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } if(requestDelimiter==null || requestDelimiter.compareTo(VALUE_OF_REQUEST_DELIMITIER)!=0){ logger.error("Path Parameter to REQUEST_DELIMITIER '"+VALUE_OF_REQUEST_DELIMITIER+"' not found"); ExceptionManager.throwBadRequestException(req, "Path Parameter '"+VALUE_OF_REQUEST_DELIMITIER+"' not found as REQUEST DELIMITER", GeonetworkResolver.class, help); } String fullURL = Util.getFullURL(req); int index = fullURL.indexOf(GeonetworkRequestFilterParameters.REQUEST_DELIMITIER); int delimiterIndex = index+GeonetworkRequestFilterParameters.REQUEST_DELIMITIER.length(); //BUILDING REMAINING PATH WITHOUT GeonetworkRequestFilterParameters.REQUEST_DELIMITIER String remainPath = null; if(delimiterIndex e = req.getParameterNames(); e.hasMoreElements();){ String p = e.nextElement(); logger.debug("param "+p + " value "+Arrays.toString(req.getParameterValues(p))); } //DEBUG BODY // String readBody = IOUtils.toString(req.getReader()); // logger.debug("doPost read body request: "+readBody); ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); String geonetworkUrl = config.getGeoNetworkEndpoint(); // SPECIFIC HANDLER FOR GEONETWORK REQUEST: /srv/en/mef.export if(remainPath!=null && remainPath.compareTo(SRV_EN_MEF_EXPORT)==0){ logger.info("In case of mef.export, perfoming a custom handler"); gnCSWlURL = geonetworkUrl + SRV_EN_MEF_EXPORT; String[] uuidValues = req.getParameterValues(UUID); if(uuidValues!=null){ String data = null; for (String uuid : uuidValues) { data = UUID+"="+uuid; } if(data!=null){ logger.debug("Writing "+data +" into byte array"); byteArray.write(data.getBytes()); }else IOUtils.copy(req.getReader(), byteArray); }else IOUtils.copy(req.getReader(), byteArray); }else{ gnCSWlURL = remainPath==null ||remainPath.isEmpty()?geonetworkUrl+"/"+CSW_SERVER:geonetworkUrl+"/"+CSW_SERVER+remainPath; IOUtils.copy(req.getReader(), byteArray); } //filterPublicMetadata = theVisibility.equals(VISIBILITY.PRV)?true:false; HTTPCallsUtils httpUtils = new HTTPCallsUtils(); //PRIVATE LAYERS if(visibility.equals(VISIBILITY.PRV)){ logger.debug("Visibility: "+VISIBILITY.PRV+" getting private layers.."); //VRE LAYERS if(mode.equals(MODE.VRE)){ logger.debug("Getting "+MODE.VRE+" layers.."); //HARVESTED LAYERS }else{ logger.debug("Getting "+MODE.HARVEST+" layers, I'm using the owner: '"+owner +"' passed as parameter to filter layer/s returned.."); if(owner==null || owner.isEmpty()){ String error = "Harvest owner is missing. It is not possible to filter layers for the request "+MODE.HARVEST + " in the scope: "+scope+", without a valid owner as input"; logger.error(error); ExceptionManager.throwBadRequestException(req, error, GeonetworkResolver.class, help); } } if(account.getUser()!=null){ boolean authorized = GNAuthentication.login(httpUtils, geonetworkUrl, account.getUser(), account. getPassword()); logger.trace("Authorized on "+geonetworkUrl +" ? "+authorized); }else logger.info("Skipping authentication, ckan user (the owner) is null"); //PUBLIC LAYERS }else{ logger.debug("Visibility: "+VISIBILITY.PUB+" getting public layers.."); //VRE LAYERS if(mode.equals(MODE.VRE)){ logger.debug("Getting "+MODE.VRE+" layers, the VRE account: "+account.getUser() +" will be used as owner user for filtering... Is it right?"); owner = account.getUser(); //HARVESTED LAYERS }else{ logger.debug("Getting "+MODE.HARVEST+" layers, I'm using the owner: '"+owner +"' passed as parameter to filter layer/s returned.."); if(owner==null || owner.isEmpty()){ String error = "Harvest owner is missing. It is not possible to filter layers for the request "+MODE.HARVEST + " in the scope: "+scope+", without a valid owner as input"; logger.error(error); ExceptionManager.throwBadRequestException(req, error, GeonetworkResolver.class, help); } } } logger.info("Sending CSW POST request to URL: "+gnCSWlURL); logger.debug("Content-Type: "+req.getContentType()); //DEBUG //logger.debug("POST - BODY : "+byteArray.toString()); InputStream in = httpUtils.post(gnCSWlURL, new ByteArrayInputStream(byteArray.toByteArray()), req.getContentType(), req.getParameterMap()); // resp.setContentType(httpUtils.getLastContentType()); // OutputStream out = resp.getOutputStream(); if(in==null){ logger.warn("Input stream returned is null, sending "+HttpServletResponse.SC_NOT_FOUND); ExceptionManager.throwNotFoundException(req, "Input stream is null to the request "+gnCSWlURL+ " with body: "+byteArray.toString(), GeonetworkResolver.class, help); } try{ ReusableInputStream reus = new ReusableInputStream(in); if(visibility.equals(VISIBILITY.PRV)){ logger.info("Private VISIBILITY performing so getting public file identifiers to apply filtering.."); FilterGetRecords filterGetRecords = new FilterGetRecords(byteArray.toString()); if(filterGetRecords.getFoundPublicIds()!=null && filterGetRecords.getFoundPublicIds().size()>0){ logger.info("I'm removing list of public IDs with "+filterGetRecords.getFoundPublicIds().size() +" item/s. Is it right?"); in = GetResponseRecordFilter.overrideResponseIdsByListIds(reus, filterGetRecords.getFoundPublicIds(), "Replaced a public UUID, please ignore"); } }else { logger.info("Public VISIBILITY perfoming check on ownership..."); Document doc = GetResponseRecordFilter.inputStreamToW3CDocument(reus); List fileIdentifiers = GetResponseRecordFilter.getTextContentStringsForTagName(doc, "gmd:fileIdentifier"); List noMatchingOwner = new ArrayList(); for (String fileId : fileIdentifiers) { String own = GetResponseRecordFilter.getMetaCategoryByFileIdentifier(fileId, config.getGeoNetworkEndpoint(),config.getAdminAccount().getUser(), config.getAdminAccount().getPassword()); //String own = GetResponseRecordFilter.getMetaOwnerNameByFileIdentifier(fileId, config.getGeoNetworkEndpoint(),config.getAdminAccount().getUser(), config.getAdminAccount().getPassword()); if(own.compareTo(owner)!=0){ logger.debug("Owner of file Identifier "+fileId+" not matching the owner passed: "+owner+", removing it.."); noMatchingOwner.add(fileId); } } if(noMatchingOwner.size()>0){ logger.info("Removing "+noMatchingOwner.size()+" layer/s not macthing the owner: "+owner); in = GetResponseRecordFilter.overrideResponseIdsByListIds(reus, noMatchingOwner, "Replaced UUID owned by another user, please ignore"); }else{ logger.info("No replace on UUIDs was applied for the owner: "+owner); in = reus; } } ReusableInputStream reusIs = new ReusableInputStream(in); //END DEBUG logger.debug("Response return Content-Type: "+httpUtils.getLastContentType()); return Response .ok(reusIs) //.header(ConstantsResolver.CONTENT_DISPOSITION,"attachment; filename = \""+fileName+"\"") .header(ConstantsResolver.CONTENT_TYPE, httpUtils.getLastContentType()).build(); //int bytes = IOUtils.copy(reusIs, out); //logger.trace("POST - RETURN : "+IOUtils.toString(reusIs)); // if(bytes==0) // logger.warn("ResponseBody is empty, returning empty resp"); }catch(Exception e){ logger.error("Error on copy the response to send to client: ", e); ExceptionManager.throwInternalErrorException(req, "Error on copy the response!", GeonetworkResolver.class, help); }finally{ IOUtils.closeQuietly(in); } } catch (IllegalArgumentException e){ logger.error("IllegalArgumentException:", e); ExceptionManager.throwBadRequestException(req, "Illegal argument to carry out the request!", GeonetworkResolver.class, help); } catch (Exception e) { logger.error("Exception:", e); String error = "Sorry, an error occurred on resolving geonetwork request with scope "+scope+". Please, contact support!"; ExceptionManager.throwInternalErrorException(req, error, GeonetworkResolver.class, help); } //An error occurred try { throw new InternalServerException(req, javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR, "Error during perform POST operation to: "+gnCSWlURL, GeonetworkResolver.class, new URI(help)); } catch (URISyntaxException e) { //silent return null; } }*/ /** * Purge remain from query string. * * @param remain_value the scope * @param queryString the query string * @return the string */ private static String purgeRemainFromQueryString(String remain_value, String queryString){ // SCOPE is: /gcube/devsec/devVRE // [INFO ] 2016-04-05 15:01:42,808 org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkResolver - // Query String is: scope=/gcube/devsec/devVRE&version=2.0.2&request=GetCapabilities&service=CSW int start = queryString.indexOf(PATH_PARAM_REMAINING+"="); if(start>=0){ int end = queryString.indexOf("&", start); if(end==-1 && queryString.length()==(PATH_PARAM_REMAINING+"="+remain_value).length()){ //SCOPE IS THE UNIQUE PARAMETER INTO QUETY STRING logger.debug("Scope is the unique parameter, returning empty query string"); return ""; }else if(end=0){ int end = queryString.indexOf("&", start); if(end==-1 && queryString.length()==(PATH_PARAM_SCOPE+"="+scope_value).length()){ //SCOPE IS THE UNIQUE PARAMETER INTO QUETY STRING logger.debug("Scope is the unique parameter, returning empty query string"); return ""; }else if(end(); logger.info("Reset of GeonetworkInstance cache perfomed!"); } /** * Gets the geonetwork instance for scope. * * @param scope the scope * @return the geonetwork instance for scope * @throws Exception the exception */ protected GeonetworkInstance getGeonetworkInstanceForScope(String scope) throws Exception{ if(cacheGNInstances==null) purgeCacheGeonetworkInstances(); if(!scope.startsWith("/")) scope="/"+scope; logger.info("Attempt to get geonetwork instance from GeonetworkInstance cache for scope: "+scope); GeonetworkInstance geoInstance = cacheGNInstances.get(scope); if(geoInstance==null){ logger.info("Cache having null GeonetworkInstance for scope "+scope+", reading by Geonetwork library..."); try { geoInstance = discoveryGeonetworkInstance(scope); cacheGNInstances.put(scope, geoInstance); logger.info("Updated GeonetworkInstance Cache adding couple: Scope "+scope+" - GeonetworkInstance "+geoInstance); } catch (Exception e) { logger.error("An error occurred on reading GeonetworkInstance for scope "+scope, e); throw new Exception("Sorry, An error occurred on reading GeonetworkInstance for scope "+scope); } }else logger.info("GeonetworkInstance cache for scope: "+scope+" is not null using it: "+geoInstance); return geoInstance; } }