gCubeSystem
/
sh-fuse-integration
15
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

git-svn-id: http://svn.research-infrastructures.eu/public/d4science/gcube/branches/data-access/sh-fuse-integration/1.0@179490 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Lucio Lelii 2019-05-20 16:42:55 +00:00
parent a752a80758
commit 0b917990a9
6 changed files with 84 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.settings/org.eclipse.core.resources.prefs Normal file
View File

@ -0,0 +1,6 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8

21
src/main/java/org/gcube/data/access/storagehub/fs/FileDownload.java
View File

@ -32,8 +32,8 @@ public class FileDownload implements SHFile{
}
public int read(Pointer buf, @size_t long size, @off_t long offset) {
logger.trace("read called with size {} and offset {} ", size, offset);
logger.trace("{} read called with size {} and offset {} ", fileItem.getName(), size, offset);
while (this.offset!=offset) {
logger.trace("going in wait ({},{})",this.offset, offset);
synchronized (monitor) {
@ -53,13 +53,13 @@ public class FileDownload implements SHFile{
try {
int read =0;
logger.trace("BEFORE: bytes to read {} and read total {} and last read {}", bytesToRead, readTotal, read);
logger.trace("{} BEFORE: bytes to read {} and read total {} and last read {}", fileItem.getName(), bytesToRead, readTotal, read);
while ((read= stream.read(mybuf, 0 , bytesToRead-readTotal))!=-1 && bytesToRead>readTotal) {
buf.put(readTotal, mybuf, 0, read);
readTotal+= read;
logger.trace("INSIDE: bytes to read {} and read total {} and last read {}", bytesToRead, readTotal, read);
logger.trace("{} INSIDE: bytes to read {} and read total {} and last read {}", fileItem.getName() , bytesToRead, readTotal, read);
}
logger.trace("AFTER: bytes to read {} and read total {} and last read {}", bytesToRead, readTotal, read);
logger.trace("{} AFTER: bytes to read {} and read total {} and last read {}", fileItem.getName() , bytesToRead, readTotal, read);
}catch (Exception e) {
logger.error("error in read",e);
@ -74,7 +74,7 @@ public class FileDownload implements SHFile{
monitor.notifyAll();
}
}
logger.trace("work finished!!! {}", readTotal);
logger.trace("{} work finished!!! {}",fileItem.getName() , readTotal);
return readTotal;
}
@ -86,6 +86,9 @@ public class FileDownload implements SHFile{
//logger.trace("file is ready "+mapPathUpload.get(path).toString());
try {
stream.close();
synchronized (monitor) {
monitor.notifyAll();
}
} catch (IOException e1) {
logger.error("error closing stream",e1);
}
@ -98,11 +101,9 @@ public class FileDownload implements SHFile{
stat.st_mode.set(FileStat.S_IFREG | 0555);
stat.st_size.set(fileItem.getContent().getSize());
stat.st_mtim.tv_sec.set(fileItem.getLastModificationTime().toInstant().getEpochSecond());
stat.st_mtim.tv_nsec.set(fileItem.getLastModificationTime().toInstant().getNano());
stat.st_ctim.tv_sec.set(fileItem.getCreationTime().toInstant().getEpochSecond());
stat.st_ctim.tv_nsec.set(fileItem.getCreationTime().toInstant().getNano());
stat.st_ctim.tv_sec.set(fileItem.getLastModificationTime().toInstant().getEpochSecond());
stat.st_atim.tv_sec.set(fileItem.getLastModificationTime().toInstant().getEpochSecond());
stat.st_atim.tv_nsec.set(fileItem.getLastModificationTime().toInstant().getNano());
//stat.st_birthtime.tv_sec.set(fileItem.getCreationTime().toInstant().getEpochSecond());
return 0;
}

5
src/main/java/org/gcube/data/access/storagehub/fs/FileUpload.java
View File

@ -55,6 +55,11 @@ public class FileUpload implements SHFile {
public int getAttr(FileStat stat) {
stat.st_mode.set(FileStat.S_IFREG | 0555);
stat.st_size.set(bytesRead);
long now = System.currentTimeMillis()/1000;
//stat.st_birthtime.tv_sec.set(now);
stat.st_mtim.tv_sec.set(now);
stat.st_ctim.tv_sec.set(now);
stat.st_atim.tv_sec.set(now);
return 0;
}

23
src/main/java/org/gcube/data/access/storagehub/fs/PathUtils.java
View File

@ -6,7 +6,9 @@ import java.util.List;
import org.cache2k.Cache;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.items.SharedFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -16,12 +18,13 @@ public class PathUtils {
private Cache<String,ItemContainer<Item>> cache;
private FolderContainer rootDirectory;
private StorageHubClient client;
public PathUtils(Cache<String, ItemContainer<Item>> cache, FolderContainer rootDirectory) {
public PathUtils(Cache<String, ItemContainer<Item>> cache, FolderContainer rootDirectory, StorageHubClient client) {
super();
this.cache = cache;
this.rootDirectory = rootDirectory;
this.client = client;
}
@ -68,9 +71,23 @@ public class PathUtils {
logger.trace("found? "+(items.size()>0));
return items.size()>0? items.get(0): null;
}
if (path.startsWith(StorageHubFS.VREFOLDERS_NAME)) {
List<ItemContainer<? extends Item>> vreFolders = client.getVREFolders().getContainers();
String vreName = path.split("/")[1];
for (ItemContainer<? extends Item> vreContainer : vreFolders) {
SharedFolder veFolder = (SharedFolder)vreContainer.get();
if (veFolder.getDisplayName().equals(vreName)) {
String nextPath = path.replace(StorageHubFS.VREFOLDERS_NAME+"/"+veFolder.getDisplayName(), "");
if(nextPath.isEmpty()) return vreContainer;
else return getPathRecursive(nextPath, (FolderContainer)vreContainer);
}
}
}
String nextName = path.substring(0, path.indexOf("/"));
String rest = path.substring(path.indexOf("/"));
for (ItemContainer<? extends Item> container : parentContainer.findByName(nextName).withContent().getContainers()) {
if (container instanceof FolderContainer) {
logger.trace("seaching path "+rest+" in "+container.get().getTitle());

55
src/main/java/org/gcube/data/access/storagehub/fs/StorageHubFS.java
View File

@ -48,7 +48,7 @@ public class StorageHubFS extends FuseStubFS {
HashMap<String, SHFile> tempFiles = new HashMap<>();
static final String VREFOLDERS_NAME= "VREFolders";
protected static final String VREFOLDERS_NAME= "VREFolders";
Cache<String,ItemContainer<Item>> cache;
@ -68,7 +68,7 @@ public class StorageHubFS extends FuseStubFS {
.expireAfterWrite(30, TimeUnit.SECONDS)
.resilienceDuration(30, TimeUnit.SECONDS)
.build();
pathUtils = new PathUtils(cache, rootDirectory);
pathUtils = new PathUtils(cache, rootDirectory, client);
}
/*
@ -82,6 +82,7 @@ public class StorageHubFS extends FuseStubFS {
logger.trace(Thread.currentThread().getName()+" ) calling write "+ size+" "+offset);
SHFile file = tempFiles.get(path);
return file.write(buf, size, offset);
}
@ -89,7 +90,7 @@ public class StorageHubFS extends FuseStubFS {
public synchronized int flush(String path, FuseFileInfo fi) {
logger.trace("called flush for "+path);
tempFiles.get(path).flush();
tempFiles.remove(path);
logger.trace("file have been removed? {}", (tempFiles.remove(path)!=null));
return 0;
}
@ -107,7 +108,14 @@ public class StorageHubFS extends FuseStubFS {
if (pathUtils.getPath(path) != null) {
return -ErrorCodes.EEXIST();
}
uploadFile(path);
return 0;
}
private void uploadFile(String path) {
final ItemContainer<? extends Item> parentContainer;
if (path.substring(1).contains("/")) {
@ -131,10 +139,8 @@ public class StorageHubFS extends FuseStubFS {
}
}
})).start();
return 0;
}
@Override
public int getattr(String path, FileStat stat) {
@ -167,6 +173,7 @@ public class StorageHubFS extends FuseStubFS {
return 0;
}
private void getAttrSHItem(ItemContainer<? extends Item> container, FileStat stat) throws IllegalArgumentException{
if (container.getType()==ContainerType.FILE) {
@ -193,10 +200,10 @@ public class StorageHubFS extends FuseStubFS {
}
stat.st_mtim.tv_sec.set(item.getLastModificationTime().toInstant().getEpochSecond());
stat.st_mtim.tv_nsec.set(item.getLastModificationTime().toInstant().getNano());
stat.st_ctim.tv_sec.set(item.getCreationTime().toInstant().getEpochSecond());
stat.st_ctim.tv_nsec.set(item.getCreationTime().toInstant().getNano());
stat.st_atim.tv_sec.set(item.getLastModificationTime().toInstant().getEpochSecond());
stat.st_atim.tv_nsec.set(item.getLastModificationTime().toInstant().getNano());
stat.st_ctim.tv_sec.set(item.getLastModificationTime().toInstant().getEpochSecond());
stat.st_ctim.tv_nsec.set(item.getLastModificationTime().toInstant().getNano());
//stat.st_atim.tv_sec.set(System.currentTimeMillis()/1000);
//stat.st_birthtime.tv_nsec.set(item.getLastModificationTime().toInstant().getNano());
}
@ -238,8 +245,8 @@ public class StorageHubFS extends FuseStubFS {
public int read(String path, Pointer buf, @size_t long size, @off_t long offset, FuseFileInfo fi) {
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(token);
logger.trace("!!! read called in path {} with size {} and offset {} ",path, size, offset);
logger.trace("!!! read called in path {} with size {} and offset {} and pointer address {}",path, size, offset, buf.address());
SHFile fileDownload;
synchronized (tempFiles) {
@ -321,10 +328,11 @@ public class StorageHubFS extends FuseStubFS {
for (ItemContainer <? extends Item> child : containers ) {
try {
Item it = child.get();
filter.apply(buf, it.getTitle(), null, 0);
String name = (it instanceof SharedFolder && ((SharedFolder)it).isVreFolder())? ((SharedFolder)it).getDisplayName() : it.getTitle();
filter.apply(buf, name, null, 0);
if (path.charAt(path.length() - 1)!='/')
path+="/";
cache.put(path+it.getTitle(), (ItemContainer<Item>) child);
cache.put(path+name, (ItemContainer<Item>) child);
}catch (Exception e) {
logger.error("error riding children ",e);
@ -471,10 +479,27 @@ public class StorageHubFS extends FuseStubFS {
@Override
public int open(String path, FuseFileInfo fi) {
logger.info("open called");
logger.info("open called {} {}",path, fi.fh.getMemory().address());
return 0;
}
@Override
public int release(String path, FuseFileInfo fi) {
logger.info("release called {} {}",path, fi.fh.getMemory().address());
return 0;
}
@Override
public int truncate(String path, long size) {
logger.info("truncate called {} ",path);
ScopeProvider.instance.set(scope);
SecurityTokenProvider.instance.set(token);
cache.remove(path);
uploadFile(path);
return 0;
}
/*
@Override
public int access(String path, int mask) {

4
src/test/java/org/gcube/data/access/storagehub/fuse/FuseTest.java
View File

@ -11,7 +11,7 @@ public class FuseTest {
@Test
public void mount() {
StorageHubFS memfs = new StorageHubFS("b7c80297-e4ed-42ab-ab42-fdc0b8b0eabf-98187548","/gcube");
StorageHubFS memfs = new StorageHubFS("7c26a682-f47b-4e6e-90e0-6d101a4314cd-980114272","/pred4s/preprod/preVRE");
try {
String path;
switch (Platform.getNativePlatform().getOS()) {
@ -19,7 +19,7 @@ public class FuseTest {
path = "J:\\";
break;
default:
path = "/home/lucio/javaMount/mnt1";
path = "/home/lucio/javaMount/mnt";
}
memfs.mount(Paths.get(path), true, true);
} finally {