585 lines
17 KiB
Java
585 lines
17 KiB
Java
package org.gcube.data.access.fs;
|
|
|
|
import java.io.BufferedInputStream;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileOutputStream;
|
|
import java.io.InputStream;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URI;
|
|
import java.net.URL;
|
|
import java.nio.file.Files;
|
|
import java.util.Arrays;
|
|
import java.util.Enumeration;
|
|
import java.util.HashSet;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
import java.util.Properties;
|
|
|
|
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
|
import org.gcube.common.scope.api.ScopeProvider;
|
|
import org.gcube.common.storagehub.client.StreamDescriptor;
|
|
import org.gcube.common.storagehub.client.dsl.ContainerType;
|
|
import org.gcube.common.storagehub.client.dsl.FileContainer;
|
|
import org.gcube.common.storagehub.client.dsl.FolderContainer;
|
|
import org.gcube.common.storagehub.client.dsl.ItemContainer;
|
|
import org.gcube.common.storagehub.client.dsl.OpenResolver;
|
|
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
|
|
import org.gcube.common.storagehub.client.dsl.VREFolderManager;
|
|
import org.gcube.common.storagehub.client.plugins.AbstractPlugin;
|
|
import org.gcube.common.storagehub.client.proxies.UserManagerClient;
|
|
import org.gcube.common.storagehub.model.Metadata;
|
|
import org.gcube.common.storagehub.model.acls.AccessType;
|
|
import org.gcube.common.storagehub.model.exceptions.StorageHubException;
|
|
import org.gcube.common.storagehub.model.items.FolderItem;
|
|
import org.gcube.common.storagehub.model.items.Item;
|
|
import org.junit.BeforeClass;
|
|
import org.junit.Test;
|
|
|
|
|
|
|
|
public class Items {
|
|
|
|
private static final String propFile = "C:\\Users\\tilli\\Documents\\eclipse\\tokens.properties";
|
|
|
|
private static final String tokens = "dev-root";
|
|
//private static final String tokens = "prod-root";
|
|
|
|
@BeforeClass
|
|
public static void setUp(){
|
|
|
|
try(InputStream is = new FileInputStream(new File(propFile))){
|
|
Properties prop = new Properties();
|
|
prop.load(is);
|
|
String value =(String)prop.get(tokens);
|
|
String[] splitValue = value.split(",");
|
|
String context = splitValue[0];
|
|
String token = splitValue[1];
|
|
SecurityTokenProvider.instance.set(token);
|
|
ScopeProvider.instance.set(context);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
}
|
|
|
|
|
|
@Test
|
|
public void addUserToVRe() throws StorageHubException {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
|
|
String vresFile = "C:\\Users\\tilli\\Downloads\\vresToAddGCat.txt";
|
|
|
|
try(InputStream is = new FileInputStream(new File(vresFile))){
|
|
Properties prop = new Properties();
|
|
prop.load(is);
|
|
Enumeration<Object> enumer = prop.keys();
|
|
while (enumer.hasMoreElements()) {
|
|
try {
|
|
String vre= (String) enumer.nextElement();
|
|
shc.getVreFolderManager(vre).addUser("gCat");
|
|
System.out.println("added to "+vre);
|
|
}catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
}catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
@Test
|
|
public void search() {
|
|
try {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
List<? extends Item> s = shc.getWSRoot().search("WD%",false).getItems();
|
|
s.forEach(i -> System.out.println(i.getName()+" "+i.getId()));
|
|
}catch (Exception e ) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
@Test
|
|
public void forceDelete() throws Exception{
|
|
|
|
StorageHubClient shc = new StorageHubClient();
|
|
//shc.open("41b904cd-128a-4121-8fd7-82498187ca06").asFolder().getAnchestors().getItems().forEach(i -> System.out.println(i.getTitle()));
|
|
shc.open("7ac99eea-d768-4864-a248-6d4ccf43d931").asFile().setDescription("new descr");
|
|
}
|
|
|
|
@Test
|
|
public void changeAcls() throws Exception{
|
|
|
|
StorageHubClient shc = new StorageHubClient();
|
|
StringBuilder builder = new StringBuilder();
|
|
//System.out.println(shc.open("65e502ff-92ef-46cc-afbd-3e91f4b680be").asFolder().canWrite());
|
|
OpenResolver openResolver= shc.open("1322e7b2-bad6-4d64-a063-db0d05b16a67");
|
|
if(openResolver.resolve().getType()==ContainerType.FILE) {
|
|
String publicLink = openResolver.asFile().getPublicLink().toString();
|
|
String itemName = openResolver.asFile().get().getTitle();
|
|
|
|
builder.append(itemName + " ("+publicLink+")");
|
|
builder.append("\n");
|
|
} else {
|
|
System.out.println("\n\n\nNON e' un FILE cosè? = " + openResolver.resolve().getType());
|
|
}
|
|
System.out.println(builder.toString());
|
|
}
|
|
|
|
|
|
@Test
|
|
public void setAdmin() throws Exception{
|
|
try {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
shc.getVreFolderManager("gcube-devsec-devVRE").setAdmin("giancarlo.panichi");
|
|
}catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
|
|
@Test
|
|
public void removeAdmin() throws Exception{
|
|
try {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
shc.getVreFolderManager("gcube-devsec-devVRE").removeAdmin("luca.frosini");
|
|
}catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@Test
|
|
public void uploadFile() throws Exception{
|
|
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
FolderContainer myRoot = shc.getWSRoot();
|
|
|
|
|
|
long start = System.currentTimeMillis();
|
|
|
|
File inFile = new File("C:\\Users\\tilli\\Downloads\\testup.zip");
|
|
try(InputStream is = new FileInputStream(inFile)){
|
|
myRoot.uploadFile(is, "testUp.zip", "file");
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
System.out.println("Response in "+(System.currentTimeMillis()-start));
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Test
|
|
public void uploadAndcopyFile() throws Exception {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
FileContainer file = null;
|
|
try(InputStream is = new FileInputStream(new File("c:\\Users\\tilli\\Downloads\\expired.png"))){
|
|
file = shc.getWSRoot().uploadFile(is, "expired/exp.png", "test");
|
|
}
|
|
|
|
|
|
|
|
//file.copy(shc.getWSRoot(), "firstCopy.jpg");
|
|
}
|
|
|
|
@Test
|
|
public void download() throws Exception {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
|
|
StreamDescriptor streamDescr = shc.open("abb59b44-e3cb-408d-a1ff-73d6d8ad2ca1").asFile().download();
|
|
System.out.println("length "+streamDescr.getContentLenght());
|
|
long start = System.currentTimeMillis();
|
|
|
|
File output = Files.createTempFile("down", streamDescr.getFileName()).toFile();
|
|
try (BufferedInputStream bi = new BufferedInputStream(streamDescr.getStream()); FileOutputStream fo = new FileOutputStream(output)){
|
|
byte[] buf = new byte[2046];
|
|
int read = -1;
|
|
while ((read=bi.read(buf))!=-1) {
|
|
fo.write(buf, 0, read);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
System.out.println("file written "+output.getAbsolutePath()+" in "+(System.currentTimeMillis()-start));
|
|
|
|
}
|
|
|
|
@Test
|
|
public void addUser() throws Exception {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
shc.createUserAccount("service-account-sergencovid19");
|
|
|
|
shc.getVreFolderManager("d4science.research-infrastructures.eu-FARM-SerGen-Covid19_Ricercatore").addUser("service-account-sergencovid19");
|
|
}
|
|
|
|
@Test
|
|
public void removeUser() throws Exception {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
shc.deleteUserAccount("_sergencovid19");
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
public void changeProp() throws Exception {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
ItemContainer item = shc.open("6399daa7-2173-4314-b4f7-2afa24eae8f8").asItem();
|
|
Metadata first = item.get().getMetadata();
|
|
first.getMap().put("lucio", "ok");
|
|
|
|
item.setMetadata(first);
|
|
|
|
Metadata second = item.get().getMetadata();
|
|
for (Entry<String, Object> entry: second.getMap().entrySet())
|
|
System.out.println(entry.getKey()+" "+entry.getValue());
|
|
|
|
|
|
second.getMap().put("lucio", null);
|
|
second.getMap().put("lelii", "0");
|
|
|
|
item.setMetadata(second);
|
|
|
|
Metadata third = item.get().getMetadata();
|
|
for (Entry<String, Object> entry: third.getMap().entrySet())
|
|
System.out.println(entry.getKey()+" "+entry.getValue());
|
|
|
|
|
|
}
|
|
|
|
/*private InputStream urlToInputStream(URL url, Map<String, String> args) {
|
|
HttpURLConnection con = null;
|
|
InputStream inputStream = null;
|
|
try {
|
|
con = (HttpURLConnection) url.openConnection();
|
|
con.setConnectTimeout(15000);
|
|
con.setReadTimeout(15000);
|
|
if (args != null) {
|
|
for (Entry<String, String> e : args.entrySet()) {
|
|
con.setRequestProperty(e.getKey(), e.getValue());
|
|
}
|
|
}
|
|
con.connect();
|
|
int responseCode = con.getResponseCode();
|
|
|
|
if (responseCode < 400 && responseCode > 299) {
|
|
String redirectUrl = con.getHeaderField("Location");
|
|
try {
|
|
URL newUrl = new URL(redirectUrl);
|
|
return urlToInputStream(newUrl, args);
|
|
} catch (MalformedURLException e) {
|
|
URL newUrl = new URL(url.getProtocol() + "://" + url.getHost() + redirectUrl);
|
|
return urlToInputStream(newUrl, args);
|
|
}
|
|
}
|
|
|
|
|
|
inputStream = con.getInputStream();
|
|
return inputStream;
|
|
} catch (Exception e) {
|
|
throw new RuntimeException(e);
|
|
}
|
|
}*/
|
|
|
|
@Test
|
|
public void uploadArchive() throws Exception {
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
URL remote = new URI("https://data.bluecloud.cineca.it/api/download/gAAAAABhaSJN8TUA71la3mKMOL9D"
|
|
+ "mioSBvOehbZlu54_jvscz8Zu3LXgqhr8RfJemd83QIh47z6TyMn3mD0OjpcG5g0qf9WUZCeW1J4btEqNObkaWv"
|
|
+ "pMhabvswweyFn1Jg4m5GpwCoKayvgsYYwjbjsGsQW5Hileiw==").toURL();
|
|
|
|
|
|
try(InputStream is = remote.openStream() ){
|
|
shc.getWSRoot().uploadArchive(is, "testUploadArchive");
|
|
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
public void getPublicLink() throws Exception{
|
|
StorageHubClient shc = new StorageHubClient();
|
|
System.out.println(shc.open("c2573eec-3942-47ec-94a7-04869e97bb69").asFile().getPublicLink());
|
|
}
|
|
|
|
@Test
|
|
public void findByName() throws Exception{
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
FolderContainer root = shc.open("2fd081dc-3b57-4e5f-aadf-38439aea4c1f").asFolder();
|
|
|
|
FolderContainer fc = root.newFolder("Computations", "description");
|
|
|
|
fc.share(new HashSet<>(Arrays.asList("giancarlo.panichi")), AccessType.READ_ONLY);
|
|
|
|
System.out.println(fc.get().getPath()+" "+fc.getId());
|
|
|
|
|
|
}
|
|
|
|
@Test
|
|
public void delete() throws Exception{
|
|
StorageHubClient shc = new StorageHubClient();
|
|
FolderContainer container = shc.getWSRoot();
|
|
try {
|
|
FolderContainer attachmentFolder = container.openByRelativePath("Attachment").asFolder();
|
|
FolderItem folder = attachmentFolder.get();
|
|
} catch (StorageHubException e) {
|
|
System.out.println("creating folder");
|
|
container.newFolder("Attachment","Folder created automatically by the System");
|
|
}
|
|
|
|
}
|
|
|
|
@Test
|
|
public void createFolderWhenNotExists() throws Exception{
|
|
StorageHubClient shc = new StorageHubClient();
|
|
FolderContainer container = shc.openVREFolder();
|
|
|
|
try {
|
|
FolderContainer attachmentFolder = container.openByRelativePath("Attachment-Lucio").asFolder();
|
|
System.out.println("fodler name is "+attachmentFolder.get().getName());
|
|
} catch (StorageHubException e) {
|
|
System.out.println("creating folder");
|
|
container.newFolder("Attachment-Lucio","Folder created automatically by the System");
|
|
}
|
|
|
|
}
|
|
|
|
@Test
|
|
public void downloadFile() throws Exception{
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
try {
|
|
for (ItemContainer<? extends Item> cont : shc.getWSRoot().list().getContainers()) {
|
|
if (cont.get().getTitle().equals("LucioShare1")) {
|
|
for (ItemContainer<? extends Item> c: ((FolderContainer)cont).list().getContainers()) {
|
|
c.get().getTitle();
|
|
}
|
|
}
|
|
}
|
|
}catch (StorageHubException e) {
|
|
e.printStackTrace();
|
|
}
|
|
try {
|
|
for (ItemContainer<? extends Item> cont : shc.getWSRoot().list().getContainers()) {
|
|
System.out.println(cont.get().getTitle());
|
|
}
|
|
}catch (StorageHubException e) {
|
|
e.printStackTrace();
|
|
}
|
|
/*
|
|
List<? extends Item> items = shc.getWSRoot().findByName("MyFo*").getItems();
|
|
|
|
for (Item item: items)
|
|
System.out.println(item.getName()+" "+item.getTitle()+" "+item.getClass().getSimpleName());
|
|
|
|
/*
|
|
URL link = shc.open("6c5fb143-3711-4097-9bdc-952e41355440").asFile().getPublicLink("1.1");
|
|
System.out.println(link.toString());
|
|
/*
|
|
File targetFile = new File("/tmp/"+descr.getFileName());
|
|
|
|
try {
|
|
java.nio.file.Files.copy(
|
|
descr.getStream(),
|
|
targetFile.toPath(),
|
|
StandardCopyOption.REPLACE_EXISTING);
|
|
} catch (IOException e) {
|
|
// TODO Auto-generated catch block
|
|
e.printStackTrace();
|
|
}
|
|
System.out.println("file created "+targetFile.getAbsolutePath());
|
|
/*
|
|
|
|
List<ItemContainer<? extends Item>> containers = shc.getWSRoot().list().getContainers();
|
|
|
|
for (ItemContainer<? extends Item> container : containers) {
|
|
if (container.getType()==ContainerType.FILE) {
|
|
FileContainer file = (FileContainer) container;
|
|
StreamDescriptor descr = file.download();
|
|
File targetFile = new File("/tmp/"+descr.getFileName());
|
|
|
|
try {
|
|
java.nio.file.Files.copy(
|
|
descr.getStream(),
|
|
targetFile.toPath(),
|
|
StandardCopyOption.REPLACE_EXISTING);
|
|
} catch (IOException e) {
|
|
// TODO Auto-generated catch block
|
|
e.printStackTrace();
|
|
}
|
|
System.out.println("file created "+targetFile.getAbsolutePath());
|
|
break;
|
|
}
|
|
}*/
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
public void downloadFolderWithExcludes() throws Exception{
|
|
StorageHubClient shc = new StorageHubClient();
|
|
|
|
|
|
StreamDescriptor streamDescr = shc.open("6eb20db1-2921-41ec-ab79-909edd9b58fd").asItem().download("05098be5-61a2-423a-b382-9399a04df11e");
|
|
|
|
File tmpFile = Files.createTempFile(streamDescr.getFileName(),"").toFile();
|
|
System.out.println(tmpFile.getAbsolutePath());
|
|
|
|
try(FileOutputStream fos = new FileOutputStream(tmpFile)){
|
|
InputStream is = streamDescr.getStream();
|
|
byte[] buf = new byte[2048];
|
|
int read= -1;
|
|
while ((read=is.read(buf))!=-1) {
|
|
fos.write(buf, 0, read);
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@Test
|
|
public void deleteUnusefulUsers() throws Exception{
|
|
final UserManagerClient client = AbstractPlugin.users().build();
|
|
List<String> users = client.getUsers();
|
|
StorageHubClient shc = new StorageHubClient();
|
|
for (String user : users)
|
|
if (user.startsWith("userm")) {
|
|
shc.deleteUserAccount(user);
|
|
System.out.println("user "+user+"removed");
|
|
}
|
|
}
|
|
|
|
|
|
@Test
|
|
public void removeGroup() throws Exception{
|
|
StorageHubClient sh = new StorageHubClient();
|
|
VREFolderManager vreMan = sh.getVreFolderManager("d4science.research-infrastructures.eu-OpenAIRE-OpenAIRE-Connect_EAB");
|
|
vreMan.removeVRE();
|
|
}
|
|
|
|
|
|
/*
|
|
|
|
static String baseUrl="http://workspace-repository1-d.d4science.org/storagehub";
|
|
|
|
|
|
|
|
public static List<? extends Item> list(OrderBy order, Path path, ItemFilter<?> ... filters){
|
|
Client client = ClientBuilder.newClient();
|
|
WebTarget webTarget = client.target(baseUrl+"/list/byPath?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548");
|
|
Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON);
|
|
List<? extends Item> r = invocationBuilder.get(ItemList.class).getItemlist();
|
|
return r;
|
|
|
|
}
|
|
|
|
public static void createFolder(){
|
|
//Client client = ClientBuilder.newClient();
|
|
Client client = ClientBuilder.newBuilder()
|
|
.register(MultiPartFeature.class).build();
|
|
WebTarget webTarget = client.target(baseUrl+"/item/create?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548");
|
|
|
|
FolderItem folder= new FolderItem();
|
|
folder.setName("My third folder");
|
|
folder.setTitle("My third title");
|
|
final MultiPart multiPart = new FormDataMultiPart()
|
|
.field("item", new ItemWrapper<FolderItem>(folder), MediaType.APPLICATION_JSON_TYPE)
|
|
/* .field("characterProfile", jsonToSend, MediaType.APPLICATION_JSON_TYPE)
|
|
.field("filename", fileToUpload.getName(), MediaType.TEXT_PLAIN_TYPE)
|
|
.bodyPart(fileDataBodyPart)*/;
|
|
/*
|
|
Response res = webTarget.request().post(Entity.entity(multiPart, multiPart.getMediaType()));
|
|
System.out.println("status is "+res.getStatus());
|
|
|
|
}
|
|
|
|
public static void create() throws Exception{
|
|
|
|
|
|
ClientConfig clientConfig = new ClientConfig();
|
|
clientConfig.property("DEFAULT_CHUNK_SIZE", 2048);
|
|
|
|
Clie
|
|
|
|
//Client client = ClientBuilder.newClient();
|
|
Client client = ClientBuilder.newClient(clientConfig)
|
|
.register(MultiPartFeature.class);
|
|
|
|
WebTarget webTarget = client.target(baseUrl+"/item/create?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548");
|
|
|
|
GenericFileItem folder= new GenericFileItem();
|
|
folder.setName("testUpload.tar.gz");
|
|
folder.setTitle("testUpload.tar.gz");
|
|
|
|
FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file", new File("/home/lucio/Downloads/testUpload.tar.gz"));
|
|
final MultiPart multiPart = new FormDataMultiPart().field("item", new ItemWrapper<GenericFileItem>(folder), MediaType.APPLICATION_JSON_TYPE)
|
|
.bodyPart(fileDataBodyPart, MediaType.APPLICATION_OCTET_STREAM_TYPE);
|
|
multiPart.close();
|
|
|
|
Response res = webTarget.request().post(Entity.entity(multiPart, multiPart.getMediaType()));
|
|
System.out.println("status is "+res.getStatus());
|
|
|
|
}
|
|
|
|
public static void get() throws Exception{
|
|
Client client = ClientBuilder.newClient();
|
|
WebTarget webTarget = client.target(baseUrl+"/item/6e9b8350-4854-4c22-8aa1-ba2d8135ad6d/download?gcube-token=950a0702-6ada-40e9-92dc-d243d1b45206-98187548");
|
|
Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_OCTET_STREAM);
|
|
Response res = invocationBuilder.get();
|
|
|
|
|
|
byte[] buf = new byte[1024];
|
|
/*while (is.read(buf)!=-1)
|
|
System.out.println("reading the buffer");
|
|
*/
|
|
|
|
/*
|
|
}
|
|
|
|
|
|
public static <T extends Item> T copy(T item, Path path){
|
|
return null;
|
|
}
|
|
|
|
public static <T extends Item> T move(T item, Path path){
|
|
return null;
|
|
}
|
|
|
|
public static <T extends Item> T unshareAll(T item){
|
|
return null;
|
|
}
|
|
*/
|
|
|
|
}
|