diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index b8947ec..2f5cc74 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,6 +1,8 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=1.8 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8b787fc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +# Changelog for org.gcube.data-access.sh-fuse-integration + +## Unreleased [v1.1.0-SNAPSHOT] 2020-09-07 + +### Fixes + +- Solved issue on data download (https://support.d4science.org/issues/19651) diff --git a/FUNDING.md b/FUNDING.md new file mode 100644 index 0000000..6fa9eac --- /dev/null +++ b/FUNDING.md @@ -0,0 +1,26 @@ +# Acknowledgments + +The projects leading to this software have received funding from a series of European Union programmes including: + +- the Sixth Framework Programme for Research and Technological Development + - [DILIGENT](https://cordis.europa.eu/project/id/004260) (grant no. 004260). +- the Seventh Framework Programme for research, technological development and demonstration + - [D4Science](https://cordis.europa.eu/project/id/212488) (grant no. 212488); + - [D4Science-II](https://cordis.europa.eu/project/id/239019) (grant no.239019); + - [ENVRI](https://cordis.europa.eu/project/id/283465) (grant no. 283465); + - [iMarine](https://cordis.europa.eu/project/id/283644) (grant no. 283644); + - [EUBrazilOpenBio](https://cordis.europa.eu/project/id/288754) (grant no. 288754). +- the H2020 research and innovation programme + - [SoBigData](https://cordis.europa.eu/project/id/654024) (grant no. 654024); + - [PARTHENOS](https://cordis.europa.eu/project/id/654119) (grant no. 654119); + - [EGI-Engage](https://cordis.europa.eu/project/id/654142) (grant no. 654142); + - [ENVRI PLUS](https://cordis.europa.eu/project/id/654182) (grant no. 654182); + - [BlueBRIDGE](https://cordis.europa.eu/project/id/675680) (grant no. 675680); + - [PerformFISH](https://cordis.europa.eu/project/id/727610) (grant no. 727610); + - [AGINFRA PLUS](https://cordis.europa.eu/project/id/731001) (grant no. 731001); + - [DESIRA](https://cordis.europa.eu/project/id/818194) (grant no. 818194); + - [ARIADNEplus](https://cordis.europa.eu/project/id/823914) (grant no. 823914); + - [RISIS 2](https://cordis.europa.eu/project/id/824091) (grant no. 824091); + - [EOSC-Pillar](https://cordis.europa.eu/project/id/857650) (grant no. 857650); + - [Blue Cloud](https://cordis.europa.eu/project/id/862409) (grant no. 862409); + - [SoBigData-PlusPlus](https://cordis.europa.eu/project/id/871042) (grant no. 871042); \ No newline at end of file diff --git a/LICENCE.md b/LICENCE.md new file mode 100644 index 0000000..3af0507 --- /dev/null +++ b/LICENCE.md @@ -0,0 +1,312 @@ +# European Union Public Licence V. 1.1 + + +EUPL © the European Community 2007 + + +This European Union Public Licence (the “EUPL”) applies to the Work or Software +(as defined below) which is provided under the terms of this Licence. Any use of +the Work, other than as authorised under this Licence is prohibited (to the +extent such use is covered by a right of the copyright holder of the Work). + +The Original Work is provided under the terms of this Licence when the Licensor +(as defined below) has placed the following notice immediately following the +copyright notice for the Original Work: + +Licensed under the EUPL V.1.1 + +or has expressed by any other mean his willingness to license under the EUPL. + + + +## 1. Definitions + +In this Licence, the following terms have the following meaning: + +- The Licence: this Licence. + +- The Original Work or the Software: the software distributed and/or + communicated by the Licensor under this Licence, available as Source Code and + also as Executable Code as the case may be. + +- Derivative Works: the works or software that could be created by the Licensee, + based upon the Original Work or modifications thereof. This Licence does not + define the extent of modification or dependence on the Original Work required + in order to classify a work as a Derivative Work; this extent is determined by + copyright law applicable in the country mentioned in Article 15. + +- The Work: the Original Work and/or its Derivative Works. + +- The Source Code: the human-readable form of the Work which is the most + convenient for people to study and modify. + +- The Executable Code: any code which has generally been compiled and which is + meant to be interpreted by a computer as a program. + +- The Licensor: the natural or legal person that distributes and/or communicates + the Work under the Licence. + +- Contributor(s): any natural or legal person who modifies the Work under the + Licence, or otherwise contributes to the creation of a Derivative Work. + +- The Licensee or “You”: any natural or legal person who makes any usage of the + Software under the terms of the Licence. + +- Distribution and/or Communication: any act of selling, giving, lending, + renting, distributing, communicating, transmitting, or otherwise making + available, on-line or off-line, copies of the Work or providing access to its + essential functionalities at the disposal of any other natural or legal + person. + + + +## 2. Scope of the rights granted by the Licence + +The Licensor hereby grants You a world-wide, royalty-free, non-exclusive, +sub-licensable licence to do the following, for the duration of copyright vested +in the Original Work: + +- use the Work in any circumstance and for all usage, reproduce the Work, modify +- the Original Work, and make Derivative Works based upon the Work, communicate +- to the public, including the right to make available or display the Work or +- copies thereof to the public and perform publicly, as the case may be, the +- Work, distribute the Work or copies thereof, lend and rent the Work or copies +- thereof, sub-license rights in the Work or copies thereof. + +Those rights can be exercised on any media, supports and formats, whether now +known or later invented, as far as the applicable law permits so. + +In the countries where moral rights apply, the Licensor waives his right to +exercise his moral right to the extent allowed by law in order to make effective +the licence of the economic rights here above listed. + +The Licensor grants to the Licensee royalty-free, non exclusive usage rights to +any patents held by the Licensor, to the extent necessary to make use of the +rights granted on the Work under this Licence. + + + +## 3. Communication of the Source Code + +The Licensor may provide the Work either in its Source Code form, or as +Executable Code. If the Work is provided as Executable Code, the Licensor +provides in addition a machine-readable copy of the Source Code of the Work +along with each copy of the Work that the Licensor distributes or indicates, in +a notice following the copyright notice attached to the Work, a repository where +the Source Code is easily and freely accessible for as long as the Licensor +continues to distribute and/or communicate the Work. + + + +## 4. Limitations on copyright + +Nothing in this Licence is intended to deprive the Licensee of the benefits from +any exception or limitation to the exclusive rights of the rights owners in the +Original Work or Software, of the exhaustion of those rights or of other +applicable limitations thereto. + + + +## 5. Obligations of the Licensee + +The grant of the rights mentioned above is subject to some restrictions and +obligations imposed on the Licensee. Those obligations are the following: + +Attribution right: the Licensee shall keep intact all copyright, patent or +trademarks notices and all notices that refer to the Licence and to the +disclaimer of warranties. The Licensee must include a copy of such notices and a +copy of the Licence with every copy of the Work he/she distributes and/or +communicates. The Licensee must cause any Derivative Work to carry prominent +notices stating that the Work has been modified and the date of modification. + +Copyleft clause: If the Licensee distributes and/or communicates copies of the +Original Works or Derivative Works based upon the Original Work, this +Distribution and/or Communication will be done under the terms of this Licence +or of a later version of this Licence unless the Original Work is expressly +distributed only under this version of the Licence. The Licensee (becoming +Licensor) cannot offer or impose any additional terms or conditions on the Work +or Derivative Work that alter or restrict the terms of the Licence. + +Compatibility clause: If the Licensee Distributes and/or Communicates Derivative +Works or copies thereof based upon both the Original Work and another work +licensed under a Compatible Licence, this Distribution and/or Communication can +be done under the terms of this Compatible Licence. For the sake of this clause, +“Compatible Licence” refers to the licences listed in the appendix attached to +this Licence. Should the Licensee’s obligations under the Compatible Licence +conflict with his/her obligations under this Licence, the obligations of the +Compatible Licence shall prevail. + +Provision of Source Code: When distributing and/or communicating copies of the +Work, the Licensee will provide a machine-readable copy of the Source Code or +indicate a repository where this Source will be easily and freely available for +as long as the Licensee continues to distribute and/or communicate the Work. + +Legal Protection: This Licence does not grant permission to use the trade names, +trademarks, service marks, or names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the copyright notice. + + + +## 6. Chain of Authorship + +The original Licensor warrants that the copyright in the Original Work granted +hereunder is owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each Contributor warrants that the copyright in the modifications he/she brings +to the Work are owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each time You accept the Licence, the original Licensor and subsequent +Contributors grant You a licence to their contributions to the Work, under the +terms of this Licence. + + + +## 7. Disclaimer of Warranty + +The Work is a work in progress, which is continuously improved by numerous +contributors. It is not a finished work and may therefore contain defects or +“bugs” inherent to this type of software development. + +For the above reason, the Work is provided under the Licence on an “as is” basis +and without warranties of any kind concerning the Work, including without +limitation merchantability, fitness for a particular purpose, absence of defects +or errors, accuracy, non-infringement of intellectual property rights other than +copyright as stated in Article 6 of this Licence. + +This disclaimer of warranty is an essential part of the Licence and a condition +for the grant of any rights to the Work. + + + +## 8. Disclaimer of Liability + +Except in the cases of wilful misconduct or damages directly caused to natural +persons, the Licensor will in no event be liable for any direct or indirect, +material or moral, damages of any kind, arising out of the Licence or of the use +of the Work, including without limitation, damages for loss of goodwill, work +stoppage, computer failure or malfunction, loss of data or any commercial +damage, even if the Licensor has been advised of the possibility of such +damage. However, the Licensor will be liable under statutory product liability +laws as far such laws apply to the Work. + + + +## 9. Additional agreements + +While distributing the Original Work or Derivative Works, You may choose to +conclude an additional agreement to offer, and charge a fee for, acceptance of +support, warranty, indemnity, or other liability obligations and/or services +consistent with this Licence. However, in accepting such obligations, You may +act only on your own behalf and on your sole responsibility, not on behalf of +the original Licensor or any other Contributor, and only if You agree to +indemnify, defend, and hold each Contributor harmless for any liability incurred +by, or claims asserted against such Contributor by the fact You have accepted +any such warranty or additional liability. + + + +## 10. Acceptance of the Licence + +The provisions of this Licence can be accepted by clicking on an icon “I agree” +placed under the bottom of a window displaying the text of this Licence or by +affirming consent in any other similar way, in accordance with the rules of +applicable law. Clicking on that icon indicates your clear and irrevocable +acceptance of this Licence and all of its terms and conditions. + +Similarly, you irrevocably accept this Licence and all of its terms and +conditions by exercising any rights granted to You by Article 2 of this Licence, +such as the use of the Work, the creation by You of a Derivative Work or the +Distribution and/or Communication by You of the Work or copies thereof. + + + +## 11. Information to the public + +In case of any Distribution and/or Communication of the Work by means of +electronic communication by You (for example, by offering to download the Work +from a remote location) the distribution channel or media (for example, a +website) must at least provide to the public the information requested by the +applicable law regarding the Licensor, the Licence and the way it may be +accessible, concluded, stored and reproduced by the Licensee. + + + +## 12. Termination of the Licence + +The Licence and the rights granted hereunder will terminate automatically upon +any breach by the Licensee of the terms of the Licence. + +Such a termination will not terminate the licences of any person who has +received the Work from the Licensee under the Licence, provided such persons +remain in full compliance with the Licence. + + + +## 13. Miscellaneous + +Without prejudice of Article 9 above, the Licence represents the complete +agreement between the Parties as to the Work licensed hereunder. + +If any provision of the Licence is invalid or unenforceable under applicable +law, this will not affect the validity or enforceability of the Licence as a +whole. Such provision will be construed and/or reformed so as necessary to make +it valid and enforceable. + +The European Commission may publish other linguistic versions and/or new +versions of this Licence, so far this is required and reasonable, without +reducing the scope of the rights granted by the Licence. New versions of the +Licence will be published with a unique version number. + +All linguistic versions of this Licence, approved by the European Commission, +have identical value. Parties can take advantage of the linguistic version of +their choice. + + + +## 14. Jurisdiction + +Any litigation resulting from the interpretation of this License, arising +between the European Commission, as a Licensor, and any Licensee, will be +subject to the jurisdiction of the Court of Justice of the European Communities, +as laid down in article 238 of the Treaty establishing the European Community. + +Any litigation arising between Parties, other than the European Commission, and +resulting from the interpretation of this License, will be subject to the +exclusive jurisdiction of the competent court where the Licensor resides or +conducts its primary business. + + + +## 15. Applicable Law + +This Licence shall be governed by the law of the European Union country where +the Licensor resides or has his registered office. + +This licence shall be governed by the Belgian law if: + +- a litigation arises between the European Commission, as a Licensor, and any +- Licensee; the Licensor, other than the European Commission, has no residence +- or registered office inside a European Union country. + + + +## Appendix + + + +“Compatible Licences” according to article 5 EUPL are: + + +- GNU General Public License (GNU GPL) v. 2 + +- Open Software License (OSL) v. 2.1, v. 3.0 + +- Common Public License v. 1.0 + +- Eclipse Public License v. 1.0 + +- Cecill v. 2.0 + diff --git a/README.md b/README.md new file mode 100644 index 0000000..a6a6b05 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +StorageHub Fuse Integration +-------------------------------------------------- + + +## Built with +* [OpenJDK](https://openjdk.java.net/) - The JDK used +* [JAX-RS](https://github.com/eclipse-ee4j/jaxrs-api) - Java™ API for RESTful Web Services +* [Jersey](https://jersey.github.io/) - JAX-RS runtime +* [Maven](https://maven.apache.org/) - Dependency Management + +## Change log + +See [CHANGELOG.md](CHANGELOG.md). + +## License + +This project is licensed under the EUPL V.1.1 License - see the [LICENSE.md](LICENSE.md) file for details. + +## About the gCube Framework +This software is part of the [gCubeFramework](https://www.gcube-system.org/ "gCubeFramework"): an +open-source software toolkit used for building and operating Hybrid Data +Infrastructures enabling the dynamic deployment of Virtual Research Environments +by favouring the realisation of reuse oriented policies. + +The projects leading to this software have received funding from a series of European Union programmes including: + +- the Sixth Framework Programme for Research and Technological Development + - DILIGENT (grant no. 004260). +- the Seventh Framework Programme for research, technological development and demonstration + - D4Science (grant no. 212488); + - D4Science-II (grant no.239019); + - ENVRI (grant no. 283465); + - iMarine(grant no. 283644); + - EUBrazilOpenBio (grant no. 288754). +- the H2020 research and innovation programme + - SoBigData (grant no. 654024); + - PARTHENOS (grant no. 654119); + - EGIEngage (grant no. 654142); + - ENVRIplus (grant no. 654182); + - BlueBRIDGE (grant no. 675680); + - PerformFish (grant no. 727610); + - AGINFRAplus (grant no. 731001); + - DESIRA (grant no. 818194); + - ARIADNEplus (grant no. 823914); + - RISIS2 (grant no. 824091); + diff --git a/pom.xml b/pom.xml index 17d7627..40239ff 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.gcube.data-access sh-fuse-integration - 1.0.2 + 1.1.0 SHFuseIntegration diff --git a/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java b/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java index bf66f2a..c4bbde8 100644 --- a/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java +++ b/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java @@ -1,16 +1,18 @@ package org.gcube.data.access.storagehub.fs; -import java.io.IOException; -import java.io.InputStream; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import org.gcube.common.storagehub.client.StreamDescriptor; import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.common.storagehub.model.items.AbstractFileItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import jnr.ffi.Pointer; -import jnr.ffi.types.off_t; -import jnr.ffi.types.size_t; import ru.serce.jnrfuse.ErrorCodes; import ru.serce.jnrfuse.struct.FileStat; @@ -18,65 +20,55 @@ public class FileDownload implements SHFile{ public static Logger logger = LoggerFactory.getLogger(FileDownload.class); - InputStream stream; AbstractFileItem fileItem; - - Object monitor = new Object(); - - long offset = 0; - - public FileDownload(FileContainer fileContainer) throws Exception { - stream = fileContainer.download().getStream(); - fileItem = fileContainer.get(); - logger.trace("FILE-DOWNLOAD initialized with {} , {}", fileItem.getName(), fileItem.getContent().getSize()); - } - + File output; - public int read(Pointer buf, long size, long offset) { - logger.trace("{} read called with size {} and offset {} ", fileItem.getName(), size, offset); + + public FileDownload(FileContainer fileContainer) throws Exception { + StreamDescriptor stream = fileContainer.download(); - while (this.offset!=offset) { - logger.trace("going in wait ({},{})",this.offset, offset); - synchronized (monitor) { - try { - monitor.wait(); - logger.trace("waking up!!!"); - } catch (InterruptedException e2) { - logger.warn("interrupt exception",e2); - } + output = Files.createTempFile("down", stream.getFileName()).toFile(); + try (BufferedInputStream bi = new BufferedInputStream(stream.getStream()); FileOutputStream fo = new FileOutputStream(output)){ + byte[] buf = new byte[2046]; + int read = -1; + while ((read=bi.read(buf))!=-1) { + fo.write(buf, 0, read); } } - + + + fileItem = fileContainer.get(); + logger.trace("FILE-DOWNLOAD with {} , {}", fileItem.getName(), fileItem.getContent().getSize()); + } + + + + @Override + public int read(Pointer buf, long size, long offset) { + RandomAccessFile raf = null; int bytesToRead = (int) (size); - byte[] mybuf = new byte[bytesToRead]; int readTotal= 0; try { - + raf = new RandomAccessFile(output, "r"); int read =0; logger.trace("{} BEFORE: bytes to read {} and read total {} and last read {}", fileItem.getName(), bytesToRead, readTotal, read); - while ((read= stream.read(mybuf, 0 , bytesToRead-readTotal))!=-1 && bytesToRead>readTotal) { + while ((read= raf.read(mybuf, 0 , bytesToRead-readTotal))!=-1 && bytesToRead>readTotal) { buf.put(readTotal, mybuf, 0, read); readTotal+= read; logger.trace("{} INSIDE: bytes to read {} and read total {} and last read {}", fileItem.getName() , bytesToRead, readTotal, read); } logger.trace("{} AFTER: bytes to read {} and read total {} and last read {}", fileItem.getName() , bytesToRead, readTotal, read); - }catch (Exception e) { logger.error("error in read",e); - try { - stream.close(); - } catch (IOException e1) {} return -ErrorCodes.ENOENT(); } finally { - this.offset = readTotal+offset; - logger.trace("setting offset to {}",this.offset); - synchronized (monitor) { - monitor.notifyAll(); - } + try{ + if (raf!=null) raf.close(); + }catch (Exception e) {} } logger.trace("{} work finished!!! {}",fileItem.getName() , readTotal); - return readTotal; + return readTotal; } @@ -84,15 +76,7 @@ public class FileDownload implements SHFile{ public synchronized int flush() { logger.trace("called flush"); - //logger.trace("file is ready "+mapPathUpload.get(path).toString()); - try { - stream.close(); - synchronized (monitor) { - monitor.notifyAll(); - } - } catch (IOException e1) { - logger.error("error closing stream",e1); - } + output.delete(); return 0; } @@ -108,4 +92,8 @@ public class FileDownload implements SHFile{ return 0; } + public File getOutputFile(){ + return output; + } + } diff --git a/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java b/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java index 9ed0f07..18ca3e5 100644 --- a/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java +++ b/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java @@ -1,10 +1,12 @@ package org.gcube.data.access.storagehub.fs; +import java.io.RandomAccessFile; import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; import java.util.Objects; +import java.util.Random; import java.util.concurrent.TimeUnit; import org.cache2k.Cache; @@ -308,9 +310,10 @@ public class StorageHubFS extends FuseStubFS { tempFiles.put(path, fileDownload); } } - + + int toReturn = fileDownload.read(buf, size, offset); - + logger.trace("!!! read ---- returning {}",toReturn); return toReturn;