From 0b917990a94104ab2cacb98248a3f6588e776b97 Mon Sep 17 00:00:00 2001 From: Lucio Lelii Date: Mon, 20 May 2019 16:42:55 +0000 Subject: [PATCH] git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/branches/data-access/sh-fuse-integration/1.0@179490 82a268e6-3cf1-43bd-a215-b396298e98cf --- .settings/org.eclipse.core.resources.prefs | 6 ++ .../access/storagehub/fs/FileDownload.java | 21 +++---- .../data/access/storagehub/fs/FileUpload.java | 5 ++ .../data/access/storagehub/fs/PathUtils.java | 23 +++++++- .../access/storagehub/fs/StorageHubFS.java | 55 ++++++++++++++----- .../data/access/storagehub/fuse/FuseTest.java | 4 +- 6 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 .settings/org.eclipse.core.resources.prefs diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java b/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java index ebda8bb..0499074 100644 --- a/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java +++ b/src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java @@ -32,8 +32,8 @@ public class FileDownload implements SHFile{ } public int read(Pointer buf, @size_t long size, @off_t long offset) { - logger.trace("read called with size {} and offset {} ", size, offset); - + logger.trace("{} read called with size {} and offset {} ", fileItem.getName(), size, offset); + while (this.offset!=offset) { logger.trace("going in wait ({},{})",this.offset, offset); synchronized (monitor) { @@ -53,13 +53,13 @@ public class FileDownload implements SHFile{ try { int read =0; - logger.trace("BEFORE: bytes to read {} and read total {} and last read {}", bytesToRead, readTotal, read); + logger.trace("{} BEFORE: bytes to read {} and read total {} and last read {}", fileItem.getName(), bytesToRead, readTotal, read); while ((read= stream.read(mybuf, 0 , bytesToRead-readTotal))!=-1 && bytesToRead>readTotal) { buf.put(readTotal, mybuf, 0, read); readTotal+= read; - logger.trace("INSIDE: bytes to read {} and read total {} and last read {}", bytesToRead, readTotal, read); + logger.trace("{} INSIDE: bytes to read {} and read total {} and last read {}", fileItem.getName() , bytesToRead, readTotal, read); } - logger.trace("AFTER: bytes to read {} and read total {} and last read {}", bytesToRead, readTotal, read); + logger.trace("{} AFTER: bytes to read {} and read total {} and last read {}", fileItem.getName() , bytesToRead, readTotal, read); }catch (Exception e) { logger.error("error in read",e); @@ -74,7 +74,7 @@ public class FileDownload implements SHFile{ monitor.notifyAll(); } } - logger.trace("work finished!!! {}", readTotal); + logger.trace("{} work finished!!! {}",fileItem.getName() , readTotal); return readTotal; } @@ -86,6 +86,9 @@ public class FileDownload implements SHFile{ //logger.trace("file is ready "+mapPathUpload.get(path).toString()); try { stream.close(); + synchronized (monitor) { + monitor.notifyAll(); + } } catch (IOException e1) { logger.error("error closing stream",e1); } @@ -98,11 +101,9 @@ public class FileDownload implements SHFile{ stat.st_mode.set(FileStat.S_IFREG | 0555); stat.st_size.set(fileItem.getContent().getSize()); stat.st_mtim.tv_sec.set(fileItem.getLastModificationTime().toInstant().getEpochSecond()); - stat.st_mtim.tv_nsec.set(fileItem.getLastModificationTime().toInstant().getNano()); - stat.st_ctim.tv_sec.set(fileItem.getCreationTime().toInstant().getEpochSecond()); - stat.st_ctim.tv_nsec.set(fileItem.getCreationTime().toInstant().getNano()); + stat.st_ctim.tv_sec.set(fileItem.getLastModificationTime().toInstant().getEpochSecond()); stat.st_atim.tv_sec.set(fileItem.getLastModificationTime().toInstant().getEpochSecond()); - stat.st_atim.tv_nsec.set(fileItem.getLastModificationTime().toInstant().getNano()); + //stat.st_birthtime.tv_sec.set(fileItem.getCreationTime().toInstant().getEpochSecond()); return 0; } diff --git a/src/main/java/org/gcube/data/access/storagehub/fs/FileUpload.java b/src/main/java/org/gcube/data/access/storagehub/fs/FileUpload.java index c1ddae2..a85ffa8 100644 --- a/src/main/java/org/gcube/data/access/storagehub/fs/FileUpload.java +++ b/src/main/java/org/gcube/data/access/storagehub/fs/FileUpload.java @@ -55,6 +55,11 @@ public class FileUpload implements SHFile { public int getAttr(FileStat stat) { stat.st_mode.set(FileStat.S_IFREG | 0555); stat.st_size.set(bytesRead); + long now = System.currentTimeMillis()/1000; + //stat.st_birthtime.tv_sec.set(now); + stat.st_mtim.tv_sec.set(now); + stat.st_ctim.tv_sec.set(now); + stat.st_atim.tv_sec.set(now); return 0; } diff --git a/src/main/java/org/gcube/data/access/storagehub/fs/PathUtils.java b/src/main/java/org/gcube/data/access/storagehub/fs/PathUtils.java index 378c5a2..773baaf 100644 --- a/src/main/java/org/gcube/data/access/storagehub/fs/PathUtils.java +++ b/src/main/java/org/gcube/data/access/storagehub/fs/PathUtils.java @@ -6,7 +6,9 @@ import java.util.List; import org.cache2k.Cache; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.ItemContainer; +import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.model.items.Item; +import org.gcube.common.storagehub.model.items.SharedFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,12 +18,13 @@ public class PathUtils { private Cache> cache; private FolderContainer rootDirectory; - + private StorageHubClient client; - public PathUtils(Cache> cache, FolderContainer rootDirectory) { + public PathUtils(Cache> cache, FolderContainer rootDirectory, StorageHubClient client) { super(); this.cache = cache; this.rootDirectory = rootDirectory; + this.client = client; } @@ -68,9 +71,23 @@ public class PathUtils { logger.trace("found? "+(items.size()>0)); return items.size()>0? items.get(0): null; } + if (path.startsWith(StorageHubFS.VREFOLDERS_NAME)) { + List> vreFolders = client.getVREFolders().getContainers(); + String vreName = path.split("/")[1]; + for (ItemContainer vreContainer : vreFolders) { + SharedFolder veFolder = (SharedFolder)vreContainer.get(); + if (veFolder.getDisplayName().equals(vreName)) { + String nextPath = path.replace(StorageHubFS.VREFOLDERS_NAME+"/"+veFolder.getDisplayName(), ""); + if(nextPath.isEmpty()) return vreContainer; + else return getPathRecursive(nextPath, (FolderContainer)vreContainer); + } + } + } + + String nextName = path.substring(0, path.indexOf("/")); String rest = path.substring(path.indexOf("/")); - + for (ItemContainer container : parentContainer.findByName(nextName).withContent().getContainers()) { if (container instanceof FolderContainer) { logger.trace("seaching path "+rest+" in "+container.get().getTitle()); diff --git a/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java b/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java index 9a3e70c..9b662b3 100644 --- a/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java +++ b/src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java @@ -48,7 +48,7 @@ public class StorageHubFS extends FuseStubFS { HashMap tempFiles = new HashMap<>(); - static final String VREFOLDERS_NAME= "VREFolders"; + protected static final String VREFOLDERS_NAME= "VREFolders"; Cache> cache; @@ -68,7 +68,7 @@ public class StorageHubFS extends FuseStubFS { .expireAfterWrite(30, TimeUnit.SECONDS) .resilienceDuration(30, TimeUnit.SECONDS) .build(); - pathUtils = new PathUtils(cache, rootDirectory); + pathUtils = new PathUtils(cache, rootDirectory, client); } /* @@ -82,6 +82,7 @@ public class StorageHubFS extends FuseStubFS { logger.trace(Thread.currentThread().getName()+" ) calling write "+ size+" "+offset); SHFile file = tempFiles.get(path); + return file.write(buf, size, offset); } @@ -89,7 +90,7 @@ public class StorageHubFS extends FuseStubFS { public synchronized int flush(String path, FuseFileInfo fi) { logger.trace("called flush for "+path); tempFiles.get(path).flush(); - tempFiles.remove(path); + logger.trace("file have been removed? {}", (tempFiles.remove(path)!=null)); return 0; } @@ -107,7 +108,14 @@ public class StorageHubFS extends FuseStubFS { if (pathUtils.getPath(path) != null) { return -ErrorCodes.EEXIST(); } + + uploadFile(path); + return 0; + } + + + private void uploadFile(String path) { final ItemContainer parentContainer; if (path.substring(1).contains("/")) { @@ -131,10 +139,8 @@ public class StorageHubFS extends FuseStubFS { } } })).start(); - - return 0; } - + @Override public int getattr(String path, FileStat stat) { @@ -167,6 +173,7 @@ public class StorageHubFS extends FuseStubFS { return 0; } + private void getAttrSHItem(ItemContainer container, FileStat stat) throws IllegalArgumentException{ if (container.getType()==ContainerType.FILE) { @@ -193,10 +200,10 @@ public class StorageHubFS extends FuseStubFS { } stat.st_mtim.tv_sec.set(item.getLastModificationTime().toInstant().getEpochSecond()); stat.st_mtim.tv_nsec.set(item.getLastModificationTime().toInstant().getNano()); - stat.st_ctim.tv_sec.set(item.getCreationTime().toInstant().getEpochSecond()); - stat.st_ctim.tv_nsec.set(item.getCreationTime().toInstant().getNano()); - stat.st_atim.tv_sec.set(item.getLastModificationTime().toInstant().getEpochSecond()); - stat.st_atim.tv_nsec.set(item.getLastModificationTime().toInstant().getNano()); + stat.st_ctim.tv_sec.set(item.getLastModificationTime().toInstant().getEpochSecond()); + stat.st_ctim.tv_nsec.set(item.getLastModificationTime().toInstant().getNano()); + //stat.st_atim.tv_sec.set(System.currentTimeMillis()/1000); + //stat.st_birthtime.tv_nsec.set(item.getLastModificationTime().toInstant().getNano()); } @@ -238,8 +245,8 @@ public class StorageHubFS extends FuseStubFS { public int read(String path, Pointer buf, @size_t long size, @off_t long offset, FuseFileInfo fi) { ScopeProvider.instance.set(scope); SecurityTokenProvider.instance.set(token); - - logger.trace("!!! read called in path {} with size {} and offset {} ",path, size, offset); + + logger.trace("!!! read called in path {} with size {} and offset {} and pointer address {}",path, size, offset, buf.address()); SHFile fileDownload; synchronized (tempFiles) { @@ -321,10 +328,11 @@ public class StorageHubFS extends FuseStubFS { for (ItemContainer child : containers ) { try { Item it = child.get(); - filter.apply(buf, it.getTitle(), null, 0); + String name = (it instanceof SharedFolder && ((SharedFolder)it).isVreFolder())? ((SharedFolder)it).getDisplayName() : it.getTitle(); + filter.apply(buf, name, null, 0); if (path.charAt(path.length() - 1)!='/') path+="/"; - cache.put(path+it.getTitle(), (ItemContainer) child); + cache.put(path+name, (ItemContainer) child); }catch (Exception e) { logger.error("error riding children ",e); @@ -471,10 +479,27 @@ public class StorageHubFS extends FuseStubFS { @Override public int open(String path, FuseFileInfo fi) { - logger.info("open called"); + logger.info("open called {} {}",path, fi.fh.getMemory().address()); + return 0; + } + + @Override + public int release(String path, FuseFileInfo fi) { + logger.info("release called {} {}",path, fi.fh.getMemory().address()); return 0; } + @Override + public int truncate(String path, long size) { + logger.info("truncate called {} ",path); + + ScopeProvider.instance.set(scope); + SecurityTokenProvider.instance.set(token); + cache.remove(path); + uploadFile(path); + return 0; + } + /* @Override public int access(String path, int mask) { diff --git a/src/test/java/org/gcube/data/access/storagehub/fuse/FuseTest.java b/src/test/java/org/gcube/data/access/storagehub/fuse/FuseTest.java index 3a5219d..2949804 100644 --- a/src/test/java/org/gcube/data/access/storagehub/fuse/FuseTest.java +++ b/src/test/java/org/gcube/data/access/storagehub/fuse/FuseTest.java @@ -11,7 +11,7 @@ public class FuseTest { @Test public void mount() { - StorageHubFS memfs = new StorageHubFS("b7c80297-e4ed-42ab-ab42-fdc0b8b0eabf-98187548","/gcube"); + StorageHubFS memfs = new StorageHubFS("7c26a682-f47b-4e6e-90e0-6d101a4314cd-980114272","/pred4s/preprod/preVRE"); try { String path; switch (Platform.getNativePlatform().getOS()) { @@ -19,7 +19,7 @@ public class FuseTest { path = "J:\\"; break; default: - path = "/home/lucio/javaMount/mnt1"; + path = "/home/lucio/javaMount/mnt"; } memfs.mount(Paths.get(path), true, true); } finally {