package org.gcube.datatransfer.resolver.services; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.datatransfer.resolver.SingleFileStreamingOutput; import org.gcube.datatransfer.resolver.gis.GeonetworkAccessParameter; import org.gcube.datatransfer.resolver.gis.GeonetworkInstance; import org.gcube.datatransfer.resolver.gis.GeonetworkServiceInterface; import org.gcube.datatransfer.resolver.gis.exception.GeonetworkInstanceException; import org.gcube.datatransfer.resolver.gis.geonetwork.FilterGetRecords; import org.gcube.datatransfer.resolver.gis.geonetwork.GNAuthentication; import org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkRequestFilterParameters.MODE; import org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkRequestFilterParameters.VISIBILITY; import org.gcube.datatransfer.resolver.gis.geonetwork.ReusableInputStream; import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.datatransfer.resolver.services.exceptions.InternalServerException; import org.gcube.datatransfer.resolver.util.GetResponseRecordFilter; import org.gcube.datatransfer.resolver.util.HTTPCallsUtils; import org.gcube.datatransfer.resolver.util.HTTPCallsUtils.HttpResponse; import org.gcube.datatransfer.resolver.util.ScopeUtil; import org.gcube.spatial.data.geonetwork.configuration.Configuration; import org.gcube.spatial.data.geonetwork.model.Account; import org.gcube.spatial.data.geonetwork.model.Account.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; /** * The Class GeonetworkResolver. * * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) * Oct 23, 2018 */ @Path("geonetwork") public class GeonetworkResolver { /** * */ public static final String REPLACED_A_PUBLIC_UUID_PLEASE_IGNORE = "Replaced a public UUID, please ignore"; public static final String REPLACED_UUID_BY_FILTER_PLEASE_IGNORE = "Replaced UUID by "+GeonetworkResolver.class.getSimpleName()+" filters, please ignore"; private static Logger logger = LoggerFactory.getLogger(GeonetworkResolver.class); public static final String PATH_PARAM_MODE = "mode"; public static final String PATH_PARAM_REQUEST_DELIMITER = "requestDelimiter"; // public static final String PATH_PARAM_OWNER = "owner"; private static final String PATH_PARAM_VISIBILITY = "visibility"; private static final String PATH_PARAM_SCOPE = "scope"; public static final String CSW_SERVER = "srv/en/csw"; public static final String VALUE_OF_REQUEST_DELIMITIER = "$$"; public static final String PATH_PARAM_REMAINPATH = "remainPath"; public static final String QUERY_PARAM_RESET_CACHE_PARAM = "resetcache"; public static final String QUERY_PARAM_RESET_CACHED_SCOPE_PARAM = "resetcachedscope"; public static final String SRV_EN_MEF_EXPORT = "/srv/en/mef.export"; //MEF Geonetwork service public static final String UUID = "uuid"; public static final String SCOPE_SEPARATOR = "|"; protected Map cacheGNInstances; private String help = "https://wiki.gcube-system.org/gcube/GCube_Resource_Catalogue#Geonetwork_Resolver"; /** * Gets the geonetwork request criteria. * Creates a request criteria from input parameter pathWithoutGN * The parameter pathWithoutGN should be an ordered string (like REST request): * SCOPE/MODE/VISIBILITY/OWNER/$$ * SCOPE must be: ROOT|VO|VRE * MODE must be: {@link MODE} * VISIBILITY must be: {@link VISIBILITY} * OWNER (is optional): filter by owner * * @param req the req * @param mode the mode * @param scope the scope * @param visibility the visibility * @param owner the owner - pass 'null' as string if no filter applied * @param requestDelimiter the request delimiter * @param resetCache the reset cache * @param resetScope the reset scope * @return the geonetwork request criteria */ @GET @Path("{"+PATH_PARAM_SCOPE+"}/{"+PATH_PARAM_MODE+"}/{"+PATH_PARAM_VISIBILITY+"}/{filterKey}/{filterValue}/$${"+PATH_PARAM_REMAINPATH+":(/[^?$]+)?}") public Response submitGet(@Context HttpServletRequest req, @PathParam(PATH_PARAM_SCOPE) @Nullable String scope, @PathParam(PATH_PARAM_MODE) @Nullable String mode, @PathParam(PATH_PARAM_VISIBILITY) @Nullable String visibility, @PathParam("filterKey") @Nullable String filterKey, @PathParam("filterValue") @Nullable String filterValue, @PathParam(PATH_PARAM_REMAINPATH) @Nullable String remainPath, @QueryParam(QUERY_PARAM_RESET_CACHE_PARAM) @Nullable String resetCache, @QueryParam(QUERY_PARAM_RESET_CACHED_SCOPE_PARAM) @Nullable String resetScope) { logger.info(this.getClass().getSimpleName()+" GET starts..."); logger.info("Params are [mode: "+mode+", scope: "+scope+", visibility: "+visibility+", filterKey: "+filterKey+", filterValue: "+filterValue+", remainPath: "+remainPath+"]"); if(scope==null || scope.isEmpty()){ logger.error("Path Parameter 'scope' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'scope'", GeonetworkResolver.class, help); } if(mode==null || mode.isEmpty()){ logger.error("Path Parameter 'scope' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'mode'", GeonetworkResolver.class, help); } scope = ScopeUtil.normalizeScope(scope, "|"); mode = mode.toUpperCase(); try{ MODE.valueOf(mode); }catch(Exception e){ List toPrint = Arrays.asList(MODE.values()); logger.error("The 'mode' parameter is wrong, Have you pass a valid parameter MODE like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'mode' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } if(visibility==null){ logger.error("Path Parameter 'visibility' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'visibility'", GeonetworkResolver.class, help); } visibility = visibility.toUpperCase(); try{ VISIBILITY.valueOf(visibility); }catch (Exception e) { List toPrint = Arrays.asList(VISIBILITY.values()); logger.error("The 'visibility' parameter is wrong, Have you pass a valid parameter VISIBILITY like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'visibility' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } if(resetCache!=null && Boolean.parseBoolean(resetCache)){ purgeCacheGeonetworkInstances(); } if(resetScope!=null && Boolean.parseBoolean(resetScope)){ resetGeonetoworkInstanceCacheForScope(scope); } logger.info("Remaining path is: "+remainPath); String gnGetlURL = null; try { GeonetworkInstance gnInstance = getGeonetworkInstanceForScope(scope); ScopeProvider.instance.set(scope); HTTPCallsUtils httpUtils = new HTTPCallsUtils(); Configuration config = gnInstance.getGeonetworkPublisher().getConfiguration(); String geonetworkUrl = config.getGeoNetworkEndpoint(); String baseURL = remainPath==null ||remainPath.isEmpty()?geonetworkUrl+"/"+CSW_SERVER:geonetworkUrl+"/"+CSW_SERVER+remainPath; logger.info("The base URL is: "+baseURL); String queryString = req.getQueryString()==null || req.getQueryString().isEmpty()?"":"?"+req.getQueryString(); gnGetlURL = baseURL+queryString; logger.info("Sending get request to URL: "+gnGetlURL); HttpResponse proxedGNResponse = httpUtils.get(gnGetlURL); switch(proxedGNResponse.getStatus()){ case HttpServletResponse.SC_OK: //Building the response InputStream streamToWrite = IOUtils.toInputStream(proxedGNResponse.getResponse()); StreamingOutput so = new SingleFileStreamingOutput(streamToWrite); logger.info("Response return Content-Type: "+httpUtils.getLastContentType()); ResponseBuilder responseBuilder = Response .ok(so) //.header(ConstantsResolver.CONTENT_DISPOSITION,"attachment; filename = \""+fileName+"\"") .header(ConstantsResolver.CONTENT_TYPE, httpUtils.getLastContentType()); return responseBuilder.build(); case HttpServletResponse.SC_FORBIDDEN: ExceptionManager.throwForbiddenException(req, "You are not authorized to perform the request "+gnGetlURL, GeonetworkResolver.class, help); break; default: ExceptionManager.throwInternalErrorException(req, "Sorry, an error occurred performing the geonetwork request "+gnGetlURL+" with scope "+scope, GeonetworkResolver.class, help); } } catch (Exception e) { logger.error("Exception:", e); String error = "Sorry, an error occurred on resolving geonetwork request with scope "+scope+". Please, contact support!"; ExceptionManager.throwInternalErrorException(req, error, GeonetworkResolver.class, help); } //An error occurred try { throw new InternalServerException(req, javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR, "Error during perform GET operation to: "+gnGetlURL, GeonetworkResolver.class, new URI(help)); } catch (URISyntaxException e) { //silent return null; } } // /* (non-Javadoc) // * @see javax.servlet.GenericServlet#init() // */ // @Override // public void init() throws ServletException { // super.init(); // timer = new Timer(true); // timer.schedule(new TimerTask() { // @Override // public void run() { // logger.info("Resetting Geonetwork configuratiors cache..."); // purgeCacheGeonetworkInstances(); // } // }, CACHE_RESET_DELAY, CACHE_RESET_TIME); // } @POST @Path("{"+PATH_PARAM_SCOPE+"}/{"+PATH_PARAM_MODE+"}/{"+PATH_PARAM_VISIBILITY+"}/{filterKey}/{filterValue}/$${"+PATH_PARAM_REMAINPATH+":(/[^?$]+)?}") public Response submitPost(@Context HttpServletRequest req, @PathParam(PATH_PARAM_SCOPE) @Nullable String scope, @PathParam(PATH_PARAM_MODE) @Nullable String mode, @PathParam(PATH_PARAM_VISIBILITY) @Nullable String visibility, @PathParam("filterKey") @Nullable String filterKey, @PathParam("filterValue") @Nullable String filterValue, @PathParam(PATH_PARAM_REMAINPATH) @Nullable String remainPath, @QueryParam(QUERY_PARAM_RESET_CACHE_PARAM) @Nullable String resetCache, @QueryParam(QUERY_PARAM_RESET_CACHED_SCOPE_PARAM) @Nullable String resetScope) { logger.info(this.getClass().getSimpleName()+" POST starts..."); if(scope==null || scope.isEmpty()){ logger.error("Path Parameter 'scope' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'scope'", GeonetworkResolver.class, help); } if(mode==null || mode.isEmpty()){ logger.error("Path Parameter 'scope' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'mode'", GeonetworkResolver.class, help); } scope = ScopeUtil.normalizeScope(scope, "|"); mode = mode.toUpperCase(); try{ MODE.valueOf(mode); }catch(Exception e){ List toPrint = Arrays.asList(MODE.values()); logger.error("The 'mode' parameter is wrong, Have you pass a valid parameter MODE like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'mode' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } if(visibility==null){ logger.error("Path Parameter 'visibility' not found"); ExceptionManager.throwBadRequestException(req, "Missing mandatory path parameter 'visibility'", GeonetworkResolver.class, help); } visibility = visibility.toUpperCase(); try{ VISIBILITY.valueOf(visibility); }catch (Exception e) { List toPrint = Arrays.asList(VISIBILITY.values()); logger.error("The 'visibility' parameter is wrong, Have you pass a valid parameter VISIBILITY like "+toPrint+"?"); ExceptionManager.throwWrongParameterException(req, "The 'visibility' parameter must be value of "+toPrint, GeonetworkResolver.class, help); } //HOW TO PASS MORE THAN ONE? Map filters = new HashMap(); if(filterKey!=null && filterValue!=null){ filters.put(filterKey, filterValue); logger.debug("Added filter parmas to map filters: "+filters); } String gnCSWlURL = null; try { GeonetworkServiceInterface gntwAccess = new GeonetworkAccessParameter(scope); GeonetworkInstance gnInstance = gntwAccess.getGeonetworkInstance(); ScopeProvider.instance.set(scope); logger.info("set scope provider "+scope); Configuration config = gnInstance.getGeonetworkPublisher().getConfiguration(); Account account = config.getScopeConfiguration().getAccounts().get(Type.CKAN); logger.info("CKAN user owner is: "+account.getUser()); // logger.info("Parameters.."); // for (Enumeration e = req.getParameterNames(); e.hasMoreElements();){ // String p = e.nextElement(); // logger.debug("param "+p + " value "+Arrays.toString(req.getParameterValues(p))); // } //DEBUG BODY // String readBody = IOUtils.toString(req.getReader()); // logger.debug("doPost read body request: "+readBody); ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); String geonetworkUrl = config.getGeoNetworkEndpoint(); // SPECIFIC HANDLER FOR GEONETWORK REQUEST: /srv/en/mef.export if(remainPath!=null && remainPath.compareTo(SRV_EN_MEF_EXPORT)==0){ logger.info("In case of mef.export, perfoming a custom handler"); gnCSWlURL = geonetworkUrl + SRV_EN_MEF_EXPORT; String[] uuidValues = req.getParameterValues(UUID); if(uuidValues!=null){ String data = null; for (String uuid : uuidValues) { data = UUID+"="+uuid; } if(data!=null){ logger.debug("Writing "+data +" into byte array"); byteArray.write(data.getBytes()); }else IOUtils.copy(req.getInputStream(), byteArray); }else IOUtils.copy(req.getInputStream(), byteArray); }else{ logger.info("IS NOT A REQUEST TO "+SRV_EN_MEF_EXPORT); gnCSWlURL = remainPath==null || remainPath.isEmpty()?geonetworkUrl+"/"+CSW_SERVER:geonetworkUrl+"/"+CSW_SERVER+remainPath; IOUtils.copy(req.getInputStream(), byteArray); } HTTPCallsUtils httpUtils = new HTTPCallsUtils(); //PRIVATE LAYERS if(visibility.equals(VISIBILITY.PRV)){ logger.info("Visibility: "+VISIBILITY.PRV+" getting private layers.."); //VRE LAYERS if(mode.equals(MODE.VRE)){ logger.info("Getting "+MODE.VRE+" layers.."); //HARVESTED LAYERS }else{ filters.put("isHarvested", "y"); logger.info("Getting "+MODE.HARVEST+" layers, I added 'isHarvested = y' to the filters ["+filters+"]"); } if(account.getUser()!=null){ boolean authorized = GNAuthentication.login(httpUtils, geonetworkUrl, account.getUser(), account. getPassword()); logger.info("Authorized on "+geonetworkUrl +" ? "+authorized); }else logger.info("Skipping authentication, ckan user (the owner) is null"); //PUBLIC LAYERS }else{ logger.info("Visibility: "+VISIBILITY.PUB+" getting public layers.."); //VRE LAYERS if(mode.equals(MODE.VRE)){ logger.info("Getting "+MODE.VRE+" layers, the VRE account: "+account.getUser() +" will be used as owner user for filtering... Is it right?"); filters.put("ownername", account.getUser()); //HARVESTED LAYERS }else{ //filters.put("isHarvested", "y"); logger.info("Getting "+MODE.HARVEST+" layers, I'm applying the filters ["+filters+"]"); } } logger.info("Sending CSW POST request to URL: "+gnCSWlURL); logger.info("Content-Type: "+req.getContentType()); //DEBUG //logger.debug("POST - BODY : "+byteArray.toString()); InputStream in = httpUtils.post(gnCSWlURL, new ByteArrayInputStream(byteArray.toByteArray()), req.getContentType(), req.getParameterMap()); if(in==null){ logger.warn("Input stream returned is null, sending "+HttpServletResponse.SC_NOT_FOUND); ExceptionManager.throwNotFoundException(req, "Input stream is null to the request "+gnCSWlURL+ " with body: "+byteArray.toString(), GeonetworkResolver.class, help); } try{ ReusableInputStream reus = new ReusableInputStream(in); if(visibility.equals(VISIBILITY.PRV)){ logger.info("Private VISIBILITY requested, retrieving public file identifiers to apply filtering.."); FilterGetRecords filterGetRecords = new FilterGetRecords(byteArray.toString()); if(filterGetRecords.getFoundPublicIds()!=null && filterGetRecords.getFoundPublicIds().size()>0){ logger.info("I'm removing list of public IDs with "+filterGetRecords.getFoundPublicIds().size() +" item/s. Is it right?"); in = GetResponseRecordFilter.overrideResponseIdsByListIds(reus, filterGetRecords.getFoundPublicIds(), REPLACED_A_PUBLIC_UUID_PLEASE_IGNORE); } } if(filters.size()>0){ logger.info("Applying filtering on geonet:info... filter/s used: "+filters); Document doc = GetResponseRecordFilter.inputStreamToW3CDocument(reus); List fileIdentifiers = GetResponseRecordFilter.getTextContentStringsForTagName(doc, "gmd:fileIdentifier"); List noMatchingFilter = new ArrayList(); for (String fileId : fileIdentifiers) { //CKECKING THE FILTERS for (String fkey : filters.keySet()) { String value = GetResponseRecordFilter.getMetadataValueByFileIdentifier(fileId, config.getGeoNetworkEndpoint(),config.getAdminAccount().getUser(), config.getAdminAccount().getPassword(), fkey); //String own = GetResponseRecordFilter.getMetaOwnerNameByFileIdentifier(fileId, config.getGeoNetworkEndpoint(),config.getAdminAccount().getUser(), config.getAdminAccount().getPassword()); String fValue = filters.get(fkey); //Either the filter KEY doesn't matching any geonet:info (value==null) //or the value of filterKey in geonet:info doesn't matching the passed filterValue (value.compareTo(fValue)!=0) if(value==null || value.compareTo(fValue)!=0){ logger.trace(fkey +" of File Identifier "+fileId+" not matching the filter: "+fkey+" with value: "+fValue+", adding it to list to remove file identifier and exit from loop.."); noMatchingFilter.add(fileId); //WHEN I ADD THE FILE IDENTIFIER TO FILTERED ID, I CAN EXIT FROM CKECKING FILTERS LOOP break; } } } if(noMatchingFilter.size()>0){ logger.info("Removing "+noMatchingFilter.size()+" layer/s that not macthing the filters: "+filters); //Document doc2 = GetResponseRecordFilter.inputStreamToW3CDocument(reus); in = GetResponseRecordFilter.overrideResponseIdsByListIds(reus, noMatchingFilter, REPLACED_UUID_BY_FILTER_PLEASE_IGNORE); //in = GetResponseRecordFilter.w3CDocumentToInputStream(doc); }else{ logger.info("No replace on UUIDs was applied from filters: "+filters); in = reus; } } ReusableInputStream reusIs = new ReusableInputStream(in); //END DEBUG logger.info("Response return Content-Type: "+httpUtils.getLastContentType()); return Response .ok(reusIs) //.header(ConstantsResolver.CONTENT_DISPOSITION,"attachment; filename = \""+fileName+"\"") .header(ConstantsResolver.CONTENT_TYPE, httpUtils.getLastContentType()).build(); }catch(Exception e){ logger.error("Error on copy the response to send to client: ", e); ExceptionManager.throwInternalErrorException(req, "Error on copy the response!", GeonetworkResolver.class, help); }finally{ IOUtils.closeQuietly(in); } } catch (IllegalArgumentException e){ logger.error("IllegalArgumentException:", e); ExceptionManager.throwBadRequestException(req, "Illegal argument to carry out the request!", GeonetworkResolver.class, help); } catch (Exception e) { logger.error("Exception:", e); String error = "Sorry, an error occurred on resolving geonetwork request with scope "+scope+". Please, contact support!"; ExceptionManager.throwInternalErrorException(req, error, GeonetworkResolver.class, help); } //An error occurred try { throw new InternalServerException(req, javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR, "Error during perform POST operation to: "+gnCSWlURL, GeonetworkResolver.class, new URI(help)); } catch (URISyntaxException e) { //silent return null; } } // /** // * Purge remain from query string. // * // * @param remain_value the scope // * @param queryString the query string // * @return the string // */ // private static String purgeRemainFromQueryString(String remain_value, String queryString){ //// SCOPE is: /gcube/devsec/devVRE //// [INFO ] 2016-04-05 15:01:42,808 org.gcube.datatransfer.resolver.gis.geonetwork.GeonetworkResolver - //// Query String is: scope=/gcube/devsec/devVRE&version=2.0.2&request=GetCapabilities&service=CSW // int start = queryString.indexOf(PATH_PARAM_REMAINPATH+"="); // // if(start>=0){ // int end = queryString.indexOf("&", start); // // if(end==-1 && queryString.length()==(PATH_PARAM_REMAINPATH+"="+remain_value).length()){ //SCOPE IS THE UNIQUE PARAMETER INTO QUETY STRING // logger.debug("Scope is the unique parameter, returning empty query string"); // return ""; // }else if(end=0){ // int end = queryString.indexOf("&", start); // // if(end==-1 && queryString.length()==(PATH_PARAM_SCOPE+"="+scope_value).length()){ //SCOPE IS THE UNIQUE PARAMETER INTO QUETY STRING // logger.debug("Scope is the unique parameter, returning empty query string"); // return ""; // }else if(end(); logger.info("Reset of GeonetworkInstance cache perfomed!"); } /** * Gets the geonetwork instance for scope. * * @param scope the scope * @return the geonetwork instance for scope * @throws Exception the exception */ protected GeonetworkInstance getGeonetworkInstanceForScope(String scope) throws Exception{ if(cacheGNInstances==null) purgeCacheGeonetworkInstances(); scope = ScopeUtil.normalizeScope(scope, "|"); logger.info("Attempt to get geonetwork instance from GeonetworkInstance cache for scope: "+scope); GeonetworkInstance geoInstance = cacheGNInstances.get(scope); if(geoInstance==null){ logger.info("Cache having null GeonetworkInstance for scope "+scope+", reading by Geonetwork library..."); try { geoInstance = discoveryGeonetworkInstance(scope); cacheGNInstances.put(scope, geoInstance); logger.info("Updated GeonetworkInstance Cache adding couple: Scope "+scope+" - GeonetworkInstance "+geoInstance); } catch (Exception e) { logger.error("An error occurred on reading GeonetworkInstance for scope "+scope, e); throw new Exception("Sorry, An error occurred on reading GeonetworkInstance for scope "+scope); } }else logger.info("GeonetworkInstance cache for scope: "+scope+" is not null using it: "+geoInstance); return geoInstance; } }