package org.gcube.data.access.fs; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.util.Arrays; import java.util.Enumeration; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.storagehub.client.StreamDescriptor; import org.gcube.common.storagehub.client.dsl.ContainerType; import org.gcube.common.storagehub.client.dsl.FileContainer; import org.gcube.common.storagehub.client.dsl.FolderContainer; import org.gcube.common.storagehub.client.dsl.ItemContainer; import org.gcube.common.storagehub.client.dsl.OpenResolver; import org.gcube.common.storagehub.client.dsl.StorageHubClient; import org.gcube.common.storagehub.client.dsl.VREFolderManager; import org.gcube.common.storagehub.client.plugins.AbstractPlugin; import org.gcube.common.storagehub.client.proxies.UserManagerClient; import org.gcube.common.storagehub.model.Metadata; import org.gcube.common.storagehub.model.acls.AccessType; import org.gcube.common.storagehub.model.exceptions.StorageHubException; import org.gcube.common.storagehub.model.items.FolderItem; import org.gcube.common.storagehub.model.items.Item; import org.junit.BeforeClass; import org.junit.Test; public class Items { private static final String propFile = "C:\\Users\\tilli\\Documents\\eclipse\\tokens.properties"; private static final String tokens = "dev-root"; //private static final String tokens = "prod-root"; @BeforeClass public static void setUp(){ try(InputStream is = new FileInputStream(new File(propFile))){ Properties prop = new Properties(); prop.load(is); String value =(String)prop.get(tokens); String[] splitValue = value.split(","); String context = splitValue[0]; String token = splitValue[1]; SecurityTokenProvider.instance.set(token); ScopeProvider.instance.set(context); } catch (Exception e) { e.printStackTrace(); } } @Test public void addUserToVRe() throws StorageHubException { StorageHubClient shc = new StorageHubClient(); String vresFile = "C:\\Users\\tilli\\Downloads\\vresToAddGCat.txt"; try(InputStream is = new FileInputStream(new File(vresFile))){ Properties prop = new Properties(); prop.load(is); Enumeration enumer = prop.keys(); while (enumer.hasMoreElements()) { try { String vre= (String) enumer.nextElement(); shc.getVreFolderManager(vre).addUser("gCat"); System.out.println("added to "+vre); }catch (Exception e) { e.printStackTrace(); } } }catch (Exception e) { e.printStackTrace(); } } @Test public void search() { try { StorageHubClient shc = new StorageHubClient(); List s = shc.getWSRoot().search("WD%",false).getItems(); s.forEach(i -> System.out.println(i.getName()+" "+i.getId())); }catch (Exception e ) { e.printStackTrace(); } } @Test public void forceDelete() throws Exception{ StorageHubClient shc = new StorageHubClient(); //shc.open("41b904cd-128a-4121-8fd7-82498187ca06").asFolder().getAnchestors().getItems().forEach(i -> System.out.println(i.getTitle())); shc.open("7ac99eea-d768-4864-a248-6d4ccf43d931").asFile().setDescription("new descr"); } @Test public void changeAcls() throws Exception{ StorageHubClient shc = new StorageHubClient(); StringBuilder builder = new StringBuilder(); //System.out.println(shc.open("65e502ff-92ef-46cc-afbd-3e91f4b680be").asFolder().canWrite()); OpenResolver openResolver= shc.open("1322e7b2-bad6-4d64-a063-db0d05b16a67"); if(openResolver.resolve().getType()==ContainerType.FILE) { String publicLink = openResolver.asFile().getPublicLink().toString(); String itemName = openResolver.asFile().get().getTitle(); builder.append(itemName + " ("+publicLink+")"); builder.append("\n"); } else { System.out.println("\n\n\nNON e' un FILE cosè? = " + openResolver.resolve().getType()); } System.out.println(builder.toString()); } @Test public void setAdmin() throws Exception{ try { StorageHubClient shc = new StorageHubClient(); shc.getVreFolderManager("gcube-devsec-devVRE").setAdmin("giancarlo.panichi"); }catch (Exception e) { e.printStackTrace(); } } @Test public void removeAdmin() throws Exception{ try { StorageHubClient shc = new StorageHubClient(); shc.getVreFolderManager("gcube-devsec-devVRE").removeAdmin("luca.frosini"); }catch (Exception e) { e.printStackTrace(); } } @Test public void uploadFile() throws Exception{ StorageHubClient shc = new StorageHubClient(); FolderContainer myRoot = shc.getWSRoot(); long start = System.currentTimeMillis(); File inFile = new File("C:\\Users\\tilli\\Downloads\\testup.zip"); try(InputStream is = new FileInputStream(inFile)){ myRoot.uploadFile(is, "testUp.zip", "file"); } catch (Exception e) { e.printStackTrace(); } System.out.println("Response in "+(System.currentTimeMillis()-start)); } @Test public void uploadAndcopyFile() throws Exception { StorageHubClient shc = new StorageHubClient(); FileContainer file = null; try(InputStream is = new FileInputStream(new File("c:\\Users\\tilli\\Downloads\\expired.png"))){ file = shc.getWSRoot().uploadFile(is, "expired/exp.png", "test"); } //file.copy(shc.getWSRoot(), "firstCopy.jpg"); } @Test public void download() throws Exception { StorageHubClient shc = new StorageHubClient(); StreamDescriptor streamDescr = shc.open("abb59b44-e3cb-408d-a1ff-73d6d8ad2ca1").asFile().download(); System.out.println("length "+streamDescr.getContentLenght()); long start = System.currentTimeMillis(); File output = Files.createTempFile("down", streamDescr.getFileName()).toFile(); try (BufferedInputStream bi = new BufferedInputStream(streamDescr.getStream()); FileOutputStream fo = new FileOutputStream(output)){ byte[] buf = new byte[2046]; int read = -1; while ((read=bi.read(buf))!=-1) { fo.write(buf, 0, read); } } System.out.println("file written "+output.getAbsolutePath()+" in "+(System.currentTimeMillis()-start)); } @Test public void addUser() throws Exception { StorageHubClient shc = new StorageHubClient(); shc.createUserAccount("service-account-sergencovid19"); shc.getVreFolderManager("d4science.research-infrastructures.eu-FARM-SerGen-Covid19_Ricercatore").addUser("service-account-sergencovid19"); } @Test public void removeUser() throws Exception { StorageHubClient shc = new StorageHubClient(); shc.deleteUserAccount("_sergencovid19"); } @Test public void changeProp() throws Exception { StorageHubClient shc = new StorageHubClient(); ItemContainer item = shc.open("6399daa7-2173-4314-b4f7-2afa24eae8f8").asItem(); Metadata first = item.get().getMetadata(); first.getMap().put("lucio", "ok"); item.setMetadata(first); Metadata second = item.get().getMetadata(); for (Entry entry: second.getMap().entrySet()) System.out.println(entry.getKey()+" "+entry.getValue()); second.getMap().put("lucio", null); second.getMap().put("lelii", "0"); item.setMetadata(second); Metadata third = item.get().getMetadata(); for (Entry entry: third.getMap().entrySet()) System.out.println(entry.getKey()+" "+entry.getValue()); } /*private InputStream urlToInputStream(URL url, Map args) { HttpURLConnection con = null; InputStream inputStream = null; try { con = (HttpURLConnection) url.openConnection(); con.setConnectTimeout(15000); con.setReadTimeout(15000); if (args != null) { for (Entry e : args.entrySet()) { con.setRequestProperty(e.getKey(), e.getValue()); } } con.connect(); int responseCode = con.getResponseCode(); if (responseCode < 400 && responseCode > 299) { String redirectUrl = con.getHeaderField("Location"); try { URL newUrl = new URL(redirectUrl); return urlToInputStream(newUrl, args); } catch (MalformedURLException e) { URL newUrl = new URL(url.getProtocol() + "://" + url.getHost() + redirectUrl); return urlToInputStream(newUrl, args); } } inputStream = con.getInputStream(); return inputStream; } catch (Exception e) { throw new RuntimeException(e); } }*/ @Test public void uploadArchive() throws Exception { StorageHubClient shc = new StorageHubClient(); URL remote = new URI("https://data.bluecloud.cineca.it/api/download/gAAAAABhaSJN8TUA71la3mKMOL9D" + "mioSBvOehbZlu54_jvscz8Zu3LXgqhr8RfJemd83QIh47z6TyMn3mD0OjpcG5g0qf9WUZCeW1J4btEqNObkaWv" + "pMhabvswweyFn1Jg4m5GpwCoKayvgsYYwjbjsGsQW5Hileiw==").toURL(); try(InputStream is = remote.openStream() ){ shc.getWSRoot().uploadArchive(is, "testUploadArchive"); } catch (Exception e) { e.printStackTrace(); } } @Test public void getPublicLink() throws Exception{ StorageHubClient shc = new StorageHubClient(); System.out.println(shc.open("c2573eec-3942-47ec-94a7-04869e97bb69").asFile().getPublicLink()); } @Test public void findByName() throws Exception{ StorageHubClient shc = new StorageHubClient(); FolderContainer root = shc.open("2fd081dc-3b57-4e5f-aadf-38439aea4c1f").asFolder(); FolderContainer fc = root.newFolder("Computations", "description"); fc.share(new HashSet<>(Arrays.asList("giancarlo.panichi")), AccessType.READ_ONLY); System.out.println(fc.get().getPath()+" "+fc.getId()); } @Test public void delete() throws Exception{ StorageHubClient shc = new StorageHubClient(); FolderContainer container = shc.getWSRoot(); try { FolderContainer attachmentFolder = container.openByRelativePath("Attachment").asFolder(); FolderItem folder = attachmentFolder.get(); } catch (StorageHubException e) { System.out.println("creating folder"); container.newFolder("Attachment","Folder created automatically by the System"); } } @Test public void createFolderWhenNotExists() throws Exception{ StorageHubClient shc = new StorageHubClient(); FolderContainer container = shc.openVREFolder(); try { FolderContainer attachmentFolder = container.openByRelativePath("Attachment-Lucio").asFolder(); System.out.println("fodler name is "+attachmentFolder.get().getName()); } catch (StorageHubException e) { System.out.println("creating folder"); container.newFolder("Attachment-Lucio","Folder created automatically by the System"); } } @Test public void downloadFile() throws Exception{ StorageHubClient shc = new StorageHubClient(); try { for (ItemContainer cont : shc.getWSRoot().list().getContainers()) { if (cont.get().getTitle().equals("LucioShare1")) { for (ItemContainer c: ((FolderContainer)cont).list().getContainers()) { c.get().getTitle(); } } } }catch (StorageHubException e) { e.printStackTrace(); } try { for (ItemContainer cont : shc.getWSRoot().list().getContainers()) { System.out.println(cont.get().getTitle()); } }catch (StorageHubException e) { e.printStackTrace(); } /* List items = shc.getWSRoot().findByName("MyFo*").getItems(); for (Item item: items) System.out.println(item.getName()+" "+item.getTitle()+" "+item.getClass().getSimpleName()); /* URL link = shc.open("6c5fb143-3711-4097-9bdc-952e41355440").asFile().getPublicLink("1.1"); System.out.println(link.toString()); /* File targetFile = new File("/tmp/"+descr.getFileName()); try { java.nio.file.Files.copy( descr.getStream(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("file created "+targetFile.getAbsolutePath()); /* List> containers = shc.getWSRoot().list().getContainers(); for (ItemContainer container : containers) { if (container.getType()==ContainerType.FILE) { FileContainer file = (FileContainer) container; StreamDescriptor descr = file.download(); File targetFile = new File("/tmp/"+descr.getFileName()); try { java.nio.file.Files.copy( descr.getStream(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("file created "+targetFile.getAbsolutePath()); break; } }*/ } @Test public void downloadFolderWithExcludes() throws Exception{ StorageHubClient shc = new StorageHubClient(); StreamDescriptor streamDescr = shc.open("6eb20db1-2921-41ec-ab79-909edd9b58fd").asItem().download("05098be5-61a2-423a-b382-9399a04df11e"); File tmpFile = Files.createTempFile(streamDescr.getFileName(),"").toFile(); System.out.println(tmpFile.getAbsolutePath()); try(FileOutputStream fos = new FileOutputStream(tmpFile)){ InputStream is = streamDescr.getStream(); byte[] buf = new byte[2048]; int read= -1; while ((read=is.read(buf))!=-1) { fos.write(buf, 0, read); } } } @Test public void deleteUnusefulUsers() throws Exception{ final UserManagerClient client = AbstractPlugin.users().build(); List users = client.getUsers(); StorageHubClient shc = new StorageHubClient(); for (String user : users) if (user.startsWith("userm")) { shc.deleteUserAccount(user); System.out.println("user "+user+"removed"); } } @Test public void removeGroup() throws Exception{ StorageHubClient sh = new StorageHubClient(); VREFolderManager vreMan = sh.getVreFolderManager("d4science.research-infrastructures.eu-OpenAIRE-OpenAIRE-Connect_EAB"); vreMan.removeVRE(); } /* static String baseUrl="http://workspace-repository1-d.d4science.org/storagehub"; public static List list(OrderBy order, Path path, ItemFilter ... filters){ Client client = ClientBuilder.newClient(); WebTarget webTarget = client.target(baseUrl+"/list/byPath?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548"); Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON); List r = invocationBuilder.get(ItemList.class).getItemlist(); return r; } public static void createFolder(){ //Client client = ClientBuilder.newClient(); Client client = ClientBuilder.newBuilder() .register(MultiPartFeature.class).build(); WebTarget webTarget = client.target(baseUrl+"/item/create?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548"); FolderItem folder= new FolderItem(); folder.setName("My third folder"); folder.setTitle("My third title"); final MultiPart multiPart = new FormDataMultiPart() .field("item", new ItemWrapper(folder), MediaType.APPLICATION_JSON_TYPE) /* .field("characterProfile", jsonToSend, MediaType.APPLICATION_JSON_TYPE) .field("filename", fileToUpload.getName(), MediaType.TEXT_PLAIN_TYPE) .bodyPart(fileDataBodyPart)*/; /* Response res = webTarget.request().post(Entity.entity(multiPart, multiPart.getMediaType())); System.out.println("status is "+res.getStatus()); } public static void create() throws Exception{ ClientConfig clientConfig = new ClientConfig(); clientConfig.property("DEFAULT_CHUNK_SIZE", 2048); Clie //Client client = ClientBuilder.newClient(); Client client = ClientBuilder.newClient(clientConfig) .register(MultiPartFeature.class); WebTarget webTarget = client.target(baseUrl+"/item/create?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548"); GenericFileItem folder= new GenericFileItem(); folder.setName("testUpload.tar.gz"); folder.setTitle("testUpload.tar.gz"); FileDataBodyPart fileDataBodyPart = new FileDataBodyPart("file", new File("/home/lucio/Downloads/testUpload.tar.gz")); final MultiPart multiPart = new FormDataMultiPart().field("item", new ItemWrapper(folder), MediaType.APPLICATION_JSON_TYPE) .bodyPart(fileDataBodyPart, MediaType.APPLICATION_OCTET_STREAM_TYPE); multiPart.close(); Response res = webTarget.request().post(Entity.entity(multiPart, multiPart.getMediaType())); System.out.println("status is "+res.getStatus()); } public static void get() throws Exception{ Client client = ClientBuilder.newClient(); WebTarget webTarget = client.target(baseUrl+"/item/6e9b8350-4854-4c22-8aa1-ba2d8135ad6d/download?gcube-token=950a0702-6ada-40e9-92dc-d243d1b45206-98187548"); Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_OCTET_STREAM); Response res = invocationBuilder.get(); byte[] buf = new byte[1024]; /*while (is.read(buf)!=-1) System.out.println("reading the buffer"); */ /* } public static T copy(T item, Path path){ return null; } public static T move(T item, Path path){ return null; } public static T unshareAll(T item){ return null; } */ }