upload file by url added and bug on set Message property solved

This commit is contained in:
lucio.lelii 2021-10-15 19:53:14 +02:00
parent 6f96bab4ed
commit 433e075326
6 changed files with 207 additions and 72 deletions

View File

@ -18,7 +18,7 @@
<groupId>org.gcube.common</groupId>
<artifactId>storagehub-client-library</artifactId>
<version>1.3.0</version>
<version>1.3.0-SNAPSHOT</version>
<name>storagehub-client-library</name>
<dependencyManagement>

View File

@ -1,6 +1,7 @@
package org.gcube.common.storagehub.client.proxies;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.util.HashMap;
@ -618,7 +619,7 @@ public class DefaultItemManager implements ItemManagerClient {
}
}
/*
@Override
public String uploadFile(InputStream stream, String parentId, String fileName, String description) throws StorageHubException {
Call<GXWebTargetAdapterRequest, String> call = new Call<GXWebTargetAdapterRequest, String>() {
@ -661,7 +662,94 @@ public class DefaultItemManager implements ItemManagerClient {
throw new RuntimeException(e1);
}
}
*/
@Override
public String uploadFile(InputStream stream, String parentId, String fileName, String description) throws StorageHubException {
Call<GXWebTargetAdapterRequest, String> call = new Call<GXWebTargetAdapterRequest, String>() {
@Override
public String call(GXWebTargetAdapterRequest manager) throws Exception {
Objects.requireNonNull(stream, "stream cannot be null");
Objects.requireNonNull(parentId, "parentId cannot be null");
Objects.requireNonNull(fileName, "parentId cannot be null");
Objects.requireNonNull(description, "parentId cannot be null");
GXWebTargetAdapterRequest myManager = manager.register(MultiPartFeature.class).path(parentId)
.path("create").path("FILE");
GXInboundResponse response =null;
Entity<InputStream> entity = Entity.entity(stream, MediaType.APPLICATION_OCTET_STREAM_TYPE);
Map<String, Object[]> params = new HashMap<>();
params.put("name", new Object[] {fileName});
params.put("description", new Object[] {description});
response = myManager.post(entity);
if (response.isErrorResponse()) {
if (response.hasException()) {
throw response.getException();
}else
throw new BackendGenericError("HTTP error code is "+response.getHTTPCode());
}
return response.getSource().readEntity(String.class);
}
};
try {
return delegate.make(call);
}catch(StorageHubException e) {
throw e;
}catch(Exception e1) {
throw new RuntimeException(e1);
}
}
@Override
public String uploadFile(URI uri, String parentId, String fileName, String description) throws StorageHubException {
Call<GXWebTargetAdapterRequest, String> call = new Call<GXWebTargetAdapterRequest, String>() {
@Override
public String call(GXWebTargetAdapterRequest manager) throws Exception {
Objects.requireNonNull(uri, "uri cannot be null");
Objects.requireNonNull(parentId, "parentId cannot be null");
Objects.requireNonNull(fileName, "parentId cannot be null");
Objects.requireNonNull(description, "parentId cannot be null");
GXWebTargetAdapterRequest myManager = manager.register(MultiPartFeature.class).path(parentId)
.path("create").path("FILE");
MultivaluedMap<String, String> formData = new MultivaluedHashMap<String, String>();
formData.add("url", uri.toString());
formData.add("name", fileName);
formData.add("description", description);
GXInboundResponse response = myManager.post(Entity.entity(formData, MediaType.APPLICATION_FORM_URLENCODED));
if (response.isErrorResponse()) {
if (response.hasException()) {
throw response.getException();
}else
throw new BackendGenericError("HTTP error code is "+response.getHTTPCode());
}
return response.getSource().readEntity(String.class);
}
};
try {
return delegate.make(call);
}catch(StorageHubException e) {
throw e;
}catch(Exception e1) {
throw new RuntimeException(e1);
}
}
@Override
public String uploadArchive(InputStream stream, String parentId, String extractionFolderName) throws StorageHubException {
Call<GXWebTargetAdapterRequest, String> call = new Call<GXWebTargetAdapterRequest, String>() {

View File

@ -101,7 +101,7 @@ public class DefaultMessageManager implements MessageManagerClient {
public Void call(GXWebTargetAdapterRequest manager) throws Exception {
GXWebTargetAdapterRequest myManager = manager.path(id).path(prop);
GXInboundResponse response = myManager.put(Entity.json(Boolean.TRUE));
GXInboundResponse response = myManager.put(Entity.json(bool));
if (response.isErrorResponse()) {
if (response.hasException())
throw response.getException();

View File

@ -1,6 +1,7 @@
package org.gcube.common.storagehub.client.proxies;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.util.List;
import java.util.Set;
@ -42,6 +43,8 @@ public interface ItemManagerClient {
String uploadFile(InputStream stream, String parentId, String fileName, String description) throws StorageHubException;
String uploadFile(URI uri, String parentId, String fileName, String description) throws StorageHubException;
String createFolder(String parentId, String name, String description, boolean hidden) throws StorageHubException;
String createURL(String parentId, String name, String description, URL url) throws StorageHubException;

View File

@ -5,21 +5,29 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.common.storagehub.client.StreamDescriptor;
import org.gcube.common.storagehub.client.dsl.ContainerType;
import org.gcube.common.storagehub.client.dsl.FileContainer;
import org.gcube.common.storagehub.client.dsl.FolderContainer;
import org.gcube.common.storagehub.client.dsl.ItemContainer;
import org.gcube.common.storagehub.client.dsl.OpenResolver;
import org.gcube.common.storagehub.client.dsl.StorageHubClient;
import org.gcube.common.storagehub.client.dsl.VREFolderManager;
import org.gcube.common.storagehub.client.plugins.AbstractPlugin;
import org.gcube.common.storagehub.client.proxies.UserManagerClient;
import org.gcube.common.storagehub.model.Metadata;
@ -36,7 +44,7 @@ public class Items {
private static final String propFile = "C:\\Users\\tilli\\Documents\\eclipse\\tokens.properties";
private static final String tokens = "dev-root";
private static final String tokens = "prod-root";
//private static final String tokens = "prod-root";
@BeforeClass
@ -57,10 +65,12 @@ public class Items {
}
@Test
public void addUserToVRe() throws StorageHubException {
StorageHubClient shc = new StorageHubClient();
String vresFile = "C:\\Users\\tilli\\Downloads\\vresToAddGCat.txt";
try(InputStream is = new FileInputStream(new File(vresFile))){
@ -85,6 +95,7 @@ public class Items {
}
@Test
public void search() {
try {
@ -108,9 +119,19 @@ public class Items {
public void changeAcls() throws Exception{
StorageHubClient shc = new StorageHubClient();
StringBuilder builder = new StringBuilder();
//System.out.println(shc.open("65e502ff-92ef-46cc-afbd-3e91f4b680be").asFolder().canWrite());
OpenResolver openResolver= shc.open("1322e7b2-bad6-4d64-a063-db0d05b16a67");
if(openResolver.resolve().getType()==ContainerType.FILE) {
String publicLink = openResolver.asFile().getPublicLink().toString();
String itemName = openResolver.asFile().get().getTitle();
System.out.println(shc.open("65e502ff-92ef-46cc-afbd-3e91f4b680be").asFolder().canWrite());
builder.append(itemName + " ("+publicLink+")");
builder.append("\n");
} else {
System.out.println("\n\n\nNON e' un FILE cosè? = " + openResolver.resolve().getType());
}
System.out.println(builder.toString());
}
@ -137,41 +158,26 @@ public class Items {
}
}
@Test
public void renameFile() throws Exception{
public void uploadFile() throws Exception{
StorageHubClient shc = new StorageHubClient();
FolderContainer myRoot = shc.getWSRoot();
myRoot.findByName("");
FolderContainer children = myRoot.newFolder("build", "build folder");
FileContainer file = null;
try(InputStream is = new FileInputStream(new File("/home/lucio/Downloads/mg.jpg"))){
file = myRoot.uploadFile(is, "mg.jpg", "file");
try(InputStream is = new FileInputStream(new File("C:\\\\Users\\\\tilli\\\\Downloads\\\\Blue-Cloud_order_2021_10_15_1.zip"))){
myRoot.uploadFile(is, "testUpload.zip", "file");
} catch (Exception e) {
e.printStackTrace();
}
FileContainer copiedFile = file.copy(children, "mg.jpg");
try(InputStream is = new FileInputStream(new File("/home/lucio/Downloads/bck.jpg"))){
file = myRoot.uploadFile(is, "mg.jpg", "file");
} catch (Exception e) {
e.printStackTrace();
}
FileContainer secodncopiedFile = file.copy(children, "mg.jpg");
}
@ -216,14 +222,23 @@ public class Items {
}
@Test
public void emptyTrash() throws Exception {
public void addUser() throws Exception {
StorageHubClient shc = new StorageHubClient();
shc.createUserAccount("service-account-sergencovid19");
shc.getVreFolderManager("d4science.research-infrastructures.eu-FARM-SerGen-Covid19_Ricercatore").addUser("service-account-sergencovid19");
}
@Test
public void removeUser() throws Exception {
StorageHubClient shc = new StorageHubClient();
shc.emptyTrash();
shc.deleteUserAccount("_sergencovid19");
}
@Test
public void changeProp() throws Exception {
StorageHubClient shc = new StorageHubClient();
@ -251,18 +266,53 @@ public class Items {
}
/*private InputStream urlToInputStream(URL url, Map<String, String> args) {
HttpURLConnection con = null;
InputStream inputStream = null;
try {
con = (HttpURLConnection) url.openConnection();
con.setConnectTimeout(15000);
con.setReadTimeout(15000);
if (args != null) {
for (Entry<String, String> e : args.entrySet()) {
con.setRequestProperty(e.getKey(), e.getValue());
}
}
con.connect();
int responseCode = con.getResponseCode();
if (responseCode < 400 && responseCode > 299) {
String redirectUrl = con.getHeaderField("Location");
try {
URL newUrl = new URL(redirectUrl);
return urlToInputStream(newUrl, args);
} catch (MalformedURLException e) {
URL newUrl = new URL(url.getProtocol() + "://" + url.getHost() + redirectUrl);
return urlToInputStream(newUrl, args);
}
}
inputStream = con.getInputStream();
return inputStream;
} catch (Exception e) {
throw new RuntimeException(e);
}
}*/
@Test
public void uploadArchive() throws Exception {
StorageHubClient shc = new StorageHubClient();
String afi = null;
URL remote = new URI("https://data.bluecloud.cineca.it/api/download/gAAAAABhaSJN8TUA71la3mKMOL9D"
+ "mioSBvOehbZlu54_jvscz8Zu3LXgqhr8RfJemd83QIh47z6TyMn3mD0OjpcG5g0qf9WUZCeW1J4btEqNObkaWv"
+ "pMhabvswweyFn1Jg4m5GpwCoKayvgsYYwjbjsGsQW5Hileiw==").toURL();
try(InputStream is = remote.openStream() ){
shc.getWSRoot().uploadArchive(is, "testUploadArchive");
try(InputStream is = new FileInputStream(new File("/tmp/down724121986692880606my new folder.zip"))){
shc.getWSRoot().uploadArchive(is, "testUpload2");
System.out.println(afi);
} catch (Exception e) {
e.printStackTrace();
}
@ -428,6 +478,12 @@ public class Items {
}
@Test
public void removeGroup() throws Exception{
StorageHubClient sh = new StorageHubClient();
VREFolderManager vreMan = sh.getVreFolderManager("d4science.research-infrastructures.eu-OpenAIRE-OpenAIRE-Connect_EAB");
vreMan.removeVRE();
}
/*

View File

@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashMap;
@ -41,11 +42,13 @@ import org.gcube.common.storagehub.model.expressions.logical.ISDescendant;
import org.gcube.common.storagehub.model.items.AbstractFileItem;
import org.gcube.common.storagehub.model.items.GCubeItem;
import org.gcube.common.storagehub.model.items.Item;
import org.gcube.common.storagehub.model.messages.Message;
import org.gcube.common.storagehub.model.query.Queries;
import org.gcube.common.storagehub.model.query.Query;
import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.message.internal.MessageBodyProviderNotFoundException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@ -79,27 +82,17 @@ public class TestCall {
@Test
public void sendMessages() throws StorageHubException {
public void sendMessagesWithAttachment() throws Exception {
MessageManagerClient client = AbstractPlugin.messages().build();
for (int i=1500; i<3000; i++ ) {
try {
Thread.sleep(200);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
client.sendMessage(Arrays.asList("andrea.rossi"), "testMessage"+i, "body for ttrtetatrtrrttrtrtrar\ndfafsdfadsfafdsdsfafdsafaddsffdsf\nfsdafdjdjskhjghshjhdahjkasdhjjhsd"
+ "\ndfsjkfdasjklòjffafadjksò\ndfskfjklaajjfadsjkòfjòfdsajòfadòjk\ndsaffadsffbdsfajfdsbsafdsjkfdsakjlsdafjfkdsjklajlkfadsjfsasjd\n"
+ "dsfjkfdsajhdsfsdfahhflhjfdkshlkjfhjafdsjhlkjkhfjkdshjkfhsjklsdah fhadfjahlfdsjfdhjfads fdsfdhsajdfsahjfdsaljhhfahafshjsfdalhlfash\n"
+ "dfsakldfjdajfdsjòdsfaj fdsajkadskk dfsjòkdsfaj sjdfsjdja aòfs jsfadfsdò saj òa fafòadsj jsafads\n"
+ "dsfjfdhdsaljkhfdsljhdfsajkhsf fdshsajkhshjksdfhjkfdsh dsfjkh h dfshj ds\n"
+ "dfjdfahjklhjhaajkshkshjdsafhldslhkdsfaljkadsfhlfjasjkhdasfjhkhfdf ads hldfsasahfdsdfshjfdsa jjhfdsh dfhjs hsajfdhdfhfdshhjdfsjhdf\n"
+ "fdsjadsjkdfahjlkdfshkjdshjkfsdhjklsadfjkhjsaslfdhjalkdsfhjfshkfjlsldshjfsdhj\ndastest\n\n\nMessage"+i, null );
System.out.println("sending message "+i);
}
client.sendMessage(Arrays.asList("massimiliano.assante"), "subject encodato ? è ", "è econdato ??", Arrays.asList("4827dd33-6eea-4bba-b78b-b98e2fd2ddfa"));
}
@Test
public void setUnread() throws Exception {
MessageManagerClient client = AbstractPlugin.messages().build();
client.setRead("f3b7ce40-28df-40b3-ad17-f0b9bd658016", false);
}
public void addUserToGroup() throws StorageHubException {
GroupManagerClient client = AbstractPlugin.groups().build();
client.addUserToGroup("andrea.rossi", "gcube-devsec-devVRE");
@ -135,6 +128,8 @@ public class TestCall {
WorkspaceManagerClient client = AbstractPlugin.workspace().build();
System.out.println(client.restoreFromTrash("4fc0a9df-9a51-42ef-98f2-06c21bd0669b", "f3d336cc-cd00-48ba-8339-2bffcbef825e"));
}
public void unshareFolder() throws Exception{
ItemManagerClient itemclient = AbstractPlugin.item().build();
@ -188,23 +183,16 @@ public class TestCall {
client.createGcubeItem(ws.getId(), item);
}
@Test
public void upload() throws Exception{
//final ItemManagerClient client = AbstractPlugin.item().at(new URI("http://workspace-repository1-d.d4science.org:8080/storagehub")).build();
final ItemManagerClient client = AbstractPlugin.item().build();
Client client = ClientBuilder.newClient();
client.register(MultiPartFeature.class).property(ClientProperties.CHUNKED_ENCODING_SIZE, 1024).property(ClientProperties.OUTBOUND_CONTENT_LENGTH_BUFFER, -1)
.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");
WebTarget target = client.target("http://workspace-repository1-d.d4science.org:8080/storagehub/workspace/items/bc1c9525-43f7-4565-b5ea-0a0f9d7853a0/create/test-upload?gcube-token=595ca591-9921-423c-bfca-f8be19f05882-98187548");
URI remote = new URI("https://data.bluecloud.cineca.it/api/download/gAAAAABhaSJN8TUA71la3mKMOL9D"
+ "mioSBvOehbZlu54_jvscz8Zu3LXgqhr8RfJemd83QIh47z6TyMn3mD0OjpcG5g0qf9WUZCeW1J4btEqNObkaWv"
+ "pMhabvswweyFn1Jg4m5GpwCoKayvgsYYwjbjsGsQW5Hileiw==");
FormDataMultiPart multipart = new FormDataMultiPart();
multipart.field("name", "test1Gb2.db");
multipart.field("description", "description");
multipart.field("file", new FileInputStream("/home/lucio/Downloads/ar_bigdata_201705.csv"), MediaType.APPLICATION_OCTET_STREAM_TYPE);
target.request().post(Entity.entity(multipart, MediaType.MULTIPART_FORM_DATA));
//client.uploadFile(new FileInputStream("/home/lucio/Downloads/test5Gb.zip"), "bc1c9525-43f7-4565-b5ea-0a0f9d7853a0", "5gb.zip", "description");
client.uploadFile(remote, "bc1c9525-43f7-4565-b5ea-0a0f9d7853a0", "5gb.zip", "description");
@ -269,16 +257,16 @@ public class TestCall {
}
@Test
public void getRecents() {
ItemManagerClient itemclient = AbstractPlugin.item().build();
WorkspaceManagerClient wsclient = AbstractPlugin.workspace().build();
List<? extends Item> items = wsclient.getRecentModifiedFilePerVre();
System.out.println("items are "+items.size());
for (Item item: items)
System.out.println(item.getName()+ " "+item.getPath());
System.out.println(item.getName()+ " "+item.getLastModificationTime().getTimeInMillis());
}
public void createFolder() throws Exception{