package org.gcube.datatransfer.resolver.services; import java.io.IOException; import java.io.InputStream; import java.util.Objects; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.HEAD; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; import org.apache.http.HttpStatus; import org.gcube.contentmanagement.blobstorage.resource.MyFile; import org.gcube.contentmanagement.blobstorage.service.IClient; import org.gcube.contentmanager.storageclient.wrapper.AccessType; import org.gcube.contentmanager.storageclient.wrapper.BackendType; import org.gcube.contentmanager.storageclient.wrapper.MemoryType; import org.gcube.contentmanager.storageclient.wrapper.StorageClient; import org.gcube.datatransfer.resolver.ConstantsResolver; import org.gcube.datatransfer.resolver.ConstantsResolver.CONTENT_DISPOSITION_VALUE; import org.gcube.datatransfer.resolver.services.error.ExceptionManager; import org.gcube.datatransfer.resolver.storage.StorageClientInstance; import org.gcube.datatransfer.resolver.storage.StorageMetadataFile; import org.gcube.datatransfer.resolver.util.SingleFileStreamingOutput; import org.gcube.datatransfer.resolver.util.ValidateContentDisposition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The Class StorageIDResolver. * * @author Francesco Mangiacrapa at ISTI-CNR (francesco.mangiacrapa@isti.cnr.it) * Dec 14, 2018 */ @Path("storage") public class StorageIDResolver { /** * */ protected static final String STORAGEID_RESOLVER = "storageid-resolver"; private static final String STORAGE_ID = "storage-id"; private static final AccessType STORAGE_ACCESS_TYPE=AccessType.PUBLIC; private static final BackendType STORAGE_BACKEND_TYPE=BackendType.MongoDB; private static final Logger LOG = LoggerFactory.getLogger(StorageIDResolver.class); private static String help = "https://wiki.gcube-system.org/gcube/URI_Resolver#STORAGE-ID_Resolver"; protected static IClient storageManagerClient; private static MemoryType memory; /** * Gets the storage id. * * @param req the req * @param storageId the storage id * @param fileName the file name * @param contentType the content type * @param validation the validation * @return the storage id * @throws WebApplicationException the web application exception */ @GET @Path("/{storage-id}") public Response getStorageId(@Context HttpServletRequest req, @PathParam(STORAGE_ID) String storageId, @QueryParam(ConstantsResolver.QUERY_PARAM_FILE_NAME) String fileName, @QueryParam(ConstantsResolver.QUERY_PARAM_CONTENT_TYPE) String contentType, @QueryParam(ConstantsResolver.QUERY_PARAM_CONTENTDISPOSITION) String contentDisposition, @QueryParam(ConstantsResolver.QUERY_PARAM_VALIDATION) boolean validation) throws WebApplicationException { LOG.info(this.getClass().getSimpleName()+" GET starts..."); //Checking the optional parameter "Content-Disposition" CONTENT_DISPOSITION_VALUE dispositionValue = CONTENT_DISPOSITION_VALUE.attachment; //Validating the Content-Disposition value dispositionValue = ValidateContentDisposition.validValue(req, this.getClass(), help, contentDisposition); try{ if(storageId==null || storageId.isEmpty()){ LOG.error(STORAGE_ID+" not found"); throw ExceptionManager.badRequestException(req, "Missing mandatory path parameter "+STORAGE_ID, StorageIDResolver.class, help); } return resolveStorageId(req, storageId, fileName, contentType, dispositionValue, validation); }catch (Exception e) { if(!(e instanceof WebApplicationException)){ //UNEXPECTED EXCEPTION managing it as WebApplicationException String error = "Error occurred on resolving the Storage ID: "+storageId+". Please, contact the support!"; if(e.getCause()!=null) error+="\n\nCaused: "+e.getCause().getMessage(); throw ExceptionManager.internalErrorException(req, error, this.getClass(), help); } //ALREADY MANAGED AS WebApplicationException LOG.error("Exception:", e); throw (WebApplicationException) e; } } /** * Resolve storage id. * * @param httpRequest the http request * @param storageId the storage id * @param fileName the file name * @param contentType the content type * @param validation the validation * @return the response * @throws Exception the exception */ protected static Response resolveStorageId(HttpServletRequest httpRequest, String storageId, String fileName, String contentType, CONTENT_DISPOSITION_VALUE contentDisposition, boolean validation) throws Exception{ LOG.info("storage-id: "+storageId+", fileName: "+fileName+", contentType: "+contentType+", validation: "+validation); //Checking mandatory parameter storageId if (storageId == null || storageId.isEmpty()) { LOG.error("storageId not found"); throw ExceptionManager.badRequestException(httpRequest, "Missing mandatory path parameter "+STORAGE_ID, StorageIDResolver.class, help); } storageId = identifyIdAndMemory(storageId); IClient iClient = getStorageClientInstance(memory); String toSEID = null; LOG.info("I'm using the storageId {}",storageId); try{ toSEID = iClient.getId(storageId); //to Storage Encrypted ID LOG.info("Decoded ID"+" = "+ toSEID); }catch(Exception e){ LOG.error("Storage Client Exception when getting file from storage: ", e); throw ExceptionManager.notFoundException(httpRequest, "Storage Client Exception when getting file from storage with id: "+storageId, StorageIDResolver.class, help); } if(toSEID==null || toSEID.isEmpty()){ LOG.error("Decrypted id for storageId: "+storageId +" is null or empty!"); throw ExceptionManager.notFoundException(httpRequest, "Error on decrypting the "+STORAGE_ID+ " '"+storageId+"'. Is it a valid id?", StorageIDResolver.class, help); } //long size = iClient.getSize().RFileById(toSEID); long size = 0; //READING THE METADATA OF FILE FROM STORAGE StorageMetadataFile metaFile = getStorageMetadataFile(iClient,toSEID); LOG.debug("MetaFile retrieved from storage? "+ (metaFile!=null)); if(metaFile!=null) { //Reading the fileName from Storage Metadata only if the passed fileName is null if(fileName==null || fileName.isEmpty()) { fileName= metaFile.getName(); LOG.debug("Read filename {} from {}", fileName, StorageMetadataFile.class.getSimpleName()); } //Reading the contentType from Storage Metadata only if the passed contentType is null if(contentType==null || contentType.isEmpty()) { contentType = metaFile.getMimeType(); LOG.debug("Read contentType {} from {}", contentType, StorageMetadataFile.class.getSimpleName()); } //Reading the content size size = metaFile.getSize(); LOG.debug("Read size {} from {}", size, StorageMetadataFile.class.getSimpleName()); } //CHECKING TO DEFAULT METADATA fileName = fileName==null || fileName.isEmpty()?ConstantsResolver.DEFAULT_FILENAME_FROM_STORAGE_MANAGER:fileName; LOG.info("filename retrieved is {}",fileName); contentType = contentType==null || contentType.isEmpty()?ConstantsResolver.DEFAULT_CONTENTTYPE_UNKNOWN_UNKNOWN:contentType; LOG.info("contentType used is {}",contentType); //Building the response InputStream streamToWrite=iClient.get().RFileAsInputStream(toSEID); //input stream StreamingOutput so = new SingleFileStreamingOutput(streamToWrite); ResponseBuilder response = Response .ok(so) .header(ConstantsResolver.CONTENT_DISPOSITION, contentDisposition.name()+"; filename=\""+fileName+"\"") .header(ConstantsResolver.CONTENT_LENGTH, size) .header("Content-Type",contentType); // if (contentType!= null) // response.header("Content-Type",contentType); return response.build(); } /** * Http do head. * * @param req the http request * @param storageId the storage id * @param hproxycheck the hproxycheck * @return the response * @throws WebApplicationException the web application exception */ @HEAD @Path("/{storage-id}") public Response httpDoHead(@Context HttpServletRequest req, @PathParam(STORAGE_ID) String storageId, @QueryParam(ConstantsResolver.HPC) boolean hproxycheck) throws WebApplicationException { LOG.info(this.getClass().getSimpleName()+" HEAD starts..."); LOG.info("The query string is: {}", req.getQueryString()); LOG.info("Query parameter {} is {}", ConstantsResolver.HPC, hproxycheck); try{ //THIS IS FOR HPROXY CHECK if(hproxycheck){ LOG.trace("returning status 200 for Hproxy check"); ResponseBuilder response = Response.status(HttpStatus.SC_OK); return response.build(); } return validationPayload(req, storageId); }catch (Exception e) { if(!(e instanceof WebApplicationException)){ //UNEXPECTED EXCEPTION managing it as WebApplicationException String error = "Error occurred on resolving the Storage ID: "+storageId+". Please, contact the support!"; if(e.getCause()!=null) error+="\n\nCaused: "+e.getCause().getMessage(); throw ExceptionManager.internalErrorException(req, error, this.getClass(), help); } //ALREADY MANAGED AS WebApplicationException LOG.error("Exception:", e); throw (WebApplicationException) e; } } /** * Validation payload. * * @param httpRequest the http request * @param storageId the storage id * @return the response * @throws Exception the exception */ protected Response validationPayload(HttpServletRequest httpRequest, String storageId) throws Exception{ LOG.info("validationPayload called"); //Checking mandatory parameter storageId if (storageId == null || storageId.isEmpty()) { LOG.warn("storageId not found"); throw ExceptionManager.badRequestException(httpRequest, "Storage Client Exception when getting file from storage with id: "+storageId, this.getClass(), help); } storageId = identifyIdAndMemory(storageId); IClient iClient = getStorageClientInstance(memory); String toSEID = null; LOG.info("I'm using the storageId {}",storageId); try{ toSEID = iClient.getId(storageId); //to Storage Encrypted ID LOG.info("Decoded ID"+" = "+ toSEID); }catch(Exception e){ LOG.error("Storage Client Exception when getting file from storage: ", e); throw ExceptionManager.notFoundException(httpRequest, "Storage Client Exception when getting file from storage with id: "+storageId, StorageIDResolver.class, help); } if(toSEID==null || toSEID.isEmpty()){ LOG.error("Decrypted id for storageId: "+storageId +" is null or empty!"); throw ExceptionManager.notFoundException(httpRequest, "Error on decrypting the "+STORAGE_ID+ " '"+storageId+"'. Is it a valid id?", StorageIDResolver.class, help); } //SETTING DEFAULT METADATA String fileName = ConstantsResolver.DEFAULT_FILENAME_FROM_STORAGE_MANAGER; String contentType = ConstantsResolver.DEFAULT_CONTENTTYPE_UNKNOWN_UNKNOWN; //READING THE METADATA OF FILE FROM STORAGE StorageMetadataFile metadataFile = getStorageMetadataFile(iClient,toSEID); LOG.debug("MetaFile retrieved from storage? "+ (metadataFile!=null)); long size = 0; if (metadataFile != null) { // Reading the fileName from Storage Metadata only if it is not null if (metadataFile.getName() != null) { fileName = metadataFile.getName(); LOG.debug("Read filename {} from {}", fileName, StorageMetadataFile.class.getSimpleName()); } // Reading the contentType from Storage Metadata only if it is not null if (metadataFile.getMimeType() != null) { contentType = metadataFile.getMimeType(); LOG.debug("Read contentType {} from {}", contentType, StorageMetadataFile.class.getSimpleName()); } size = metadataFile.getSize(); LOG.debug("Read size {} from {}", size, StorageMetadataFile.class.getSimpleName()); } //Building the response InputStream streamToWrite=iClient.get().RFileAsInputStream(toSEID); //input stream byte[] bytes = new byte[1]; //1B int c; ResponseBuilder response = null; try { c = streamToWrite.read(bytes); LOG.info(c+" byte read from InputStream"); if(c>0){ LOG.info("at least 1 byte read, returning status 200"); IOUtils.closeQuietly(streamToWrite); response = Response.status(HttpStatus.SC_OK); }else throw ExceptionManager.notFoundException(httpRequest, "The file with id: "+storageId+" is missing in the storage", StorageIDResolver.class, help); } catch (IOException e2) { LOG.error("Error on validating the file: ",e2); throw ExceptionManager.internalErrorException(httpRequest, "Error on validating the file with id: "+storageId, StorageIDResolver.class, help); } if(response==null) throw ExceptionManager.internalErrorException(httpRequest, "Error on validating the file with id: "+storageId, StorageIDResolver.class, help); response = response .header("Content-Type", contentType) .header(ConstantsResolver.CONTENT_LENGTH, size) .header(ConstantsResolver.CONTENT_DISPOSITION,"attachment; filename=\""+fileName+"\""); iClient.forceClose(); return response.build(); } /** * Get a storage-manager client instance. A new storage instance will be instantied only if the memory type is not the same of the previous one. * In this case the old connection will be closed. * * @param memoryType indicates the memory type (Persistent, Volatile or whatever) * @return the storage client instance */ protected static IClient getStorageClientInstance(MemoryType memory) { if ((!Objects.isNull(storageManagerClient)) && (storageManagerClient.getGcubeMemoryType().equals(memory))) return storageManagerClient; else { if(!Objects.isNull(storageManagerClient)) { LOG.debug("going to close an old storage-client instance"); storageManagerClient.forceClose(); } LOG.debug("initializing a new storage-client instance"); storageManagerClient=new StorageClient(StorageIDResolver.class.getName(), StorageIDResolver.class.getSimpleName(), STORAGEID_RESOLVER, STORAGE_ACCESS_TYPE, memory, STORAGE_BACKEND_TYPE).getClient(); return storageManagerClient; } } /** * Set the right memory type extrapolated from the storageid * @param storageId storageid with backend type specification (if present) * @return storageId without backend type specification */ private static String identifyIdAndMemory(String storageId) { if(storageId.endsWith(org.gcube.contentmanagement.blobstorage.transport.backend.util.Costants.VOLATILE_URL_IDENTIFICATOR)){ LOG.info("identificated a VOLATILE url"); memory=MemoryType.VOLATILE; storageId=storageId.replace(org.gcube.contentmanagement.blobstorage.transport.backend.util.Costants.VOLATILE_URL_IDENTIFICATOR, ""); }else { memory=MemoryType.PERSISTENT; } return storageId; } /** * Gets the storage metadata file. * * @param iClient the i client * @param toSEID the to SEID * @return the storage metadata file */ public static StorageMetadataFile getStorageMetadataFile(IClient iClient, String toSEID) { try{ if(iClient==null) throw new Exception("The "+IClient.class.getName()+" is not instancied correclty. It is null!"); if(toSEID==null || toSEID.isEmpty()) throw new Exception("Invalid input identifier. The input Storage Encrypted ID is null or empty!"); long size = iClient.getSize().RFileById(toSEID); MyFile myFile = iClient.getMetaFile().RFile(toSEID); return new StorageMetadataFile(myFile, size); }catch (Exception e) { LOG.warn("Error on getting file metadata from storage, printing this warning and trying to continue..", e); return null; } } }