MaDgIK
/
openaire-library
13
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data-transfer-v2 #6

Merged
konstantina.galouni merged 26 commits from data-transfer-v2 into develop 2023-06-13 16:20:04 +02:00
Conversation 0 Commits 26 Files Changed 6 +93 -36
2 changed files with 93 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit aa5425d091 - Show all commits

18
utils/dataTransfer/transferData.component.html
View File

@ -10,8 +10,8 @@
<!-- This is the modal --> <!-- This is the modal -->
<modal-alert #egiTransferModal large="true" [okDisabled]="destinationPath.length == 0 || status == 'succeeded' <modal-alert #egiTransferModal large="true" [okDisabled]="destinationPath.length == 0 || status == 'succeeded'
||status == 'active' || !validatePath() || !validateDestinationUrl() || (!this.downloadElements || this.downloadElements.length == 0)" || (requests > 0) || !validatePath() || !validateDestinationUrl() || (!this.downloadElements || this.downloadElements.length == 0)"
(alertOutput)="transfer()" > (alertOutput)="transfer()" (cancelOutput)="init()">
<div *ngIf="!accessToken" class=""> <div *ngIf="!accessToken" class="">
<div class="uk-width-1-1 uk-margin-top uk-margin-bottom uk-text-center"> <div class="uk-width-1-1 uk-margin-top uk-margin-bottom uk-text-center">
In order to send data to a Cloud Storage, you would need to be authenticated, please login via EGI check-in. In order to send data to a Cloud Storage, you would need to be authenticated, please login via EGI check-in.
@ -51,19 +51,17 @@
<!-- /Demonstrators/EOSC-Future/EGI/--> <!-- /Demonstrators/EOSC-Future/EGI/-->
<!-- <br>--> <!-- <br>-->
<!-- --> <!-- -->
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Please select the <p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Destination storage type:</p>
Destination Storage type:</p>
<div input type="select" [(value)]="selectedDestination" hint="Select..." <div input type="select" [(value)]="selectedDestination" hint="Select..."
[options]="destinationOptions" (valueChange)="folders = {}"></div> [options]="destinationOptions" (valueChange)="folders = {}"></div>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Specify the destination system (e.g. hostname:8080):</p> <p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Destination system (e.g. hostname:8080):</p>
<div input [(value)]="destinationUrl" <div input [(value)]="destinationUrl"
[validators]="URLValidators" class=""></div> [validators]="URLValidators" class=""></div>
<ng-container <ng-container
*ngIf="selectedDestination.authType == 'password' || selectedDestination.authType == 'keys'"> *ngIf="selectedDestination.authType == 'password' || selectedDestination.authType == 'keys'">
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Provide <p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Authentication:</p>
authentication if needed:</p>
<div class="uk-grid uk-child-width-1-2"> <div class="uk-grid uk-child-width-1-2">
<div input [(value)]="destinationAuthUser" [placeholder]="'Give ' + (selectedDestination.authType == <div input [(value)]="destinationAuthUser" [placeholder]="'Give ' + (selectedDestination.authType ==
'password'? 'username':'access key') " 'password'? 'username':'access key') "
@ -74,7 +72,7 @@
</div> </div>
</ng-container> </ng-container>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Specify the destination path (e.g. /folder1/folder2):</p> <p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Destination path (e.g. /folder1/folder2):</p>
<div input [(value)]="destinationPath" <div input [(value)]="destinationPath"
[validators]="pathValidators" class=""></div> [validators]="pathValidators" class=""></div>
<!-- <div *ngIf="selectedDestination.hasBrowse">--> <!-- <div *ngIf="selectedDestination.hasBrowse">-->
@ -100,7 +98,7 @@
<!-- <a *ngIf="status != 'succeeded' && status != 'active'" (click)="transfer()">--> <!-- <a *ngIf="status != 'succeeded' && status != 'active'" (click)="transfer()">-->
<!-- Try again!--> <!-- Try again!-->
<!-- </a>--> <!-- </a>-->
<div *ngIf="status == 'active'" class="uk-flex uk-flex-center uk-text-muted"> <div *ngIf="requests > 0" class="uk-flex uk-flex-center uk-text-muted">
<div> <div>
<span class="uk-icon uk-spinner"> <span class="uk-icon uk-spinner">
<svg width="60" height="60" viewBox="0 0 30 30" xmlns="http://www.w3.org/2000/svg" <svg width="60" height="60" viewBox="0 0 30 30" xmlns="http://www.w3.org/2000/svg"
@ -109,7 +107,7 @@
</span> </span>
</div> </div>
</div> </div>
<a *ngIf="status != 'active'" class="uk-alert-close" uk-close></a> <a *ngIf="requests <= 0" class="uk-alert-close" uk-close></a>
</div> </div>
<!-- <div *ngIf="jobId || status == 'canceled'"> <!-- <div *ngIf="jobId || status == 'canceled'">
<button *ngIf=" status != 'canceled'" class="uk-button uk-button-default" (click)="getStatus()">Check status</button> <button *ngIf=" status != 'canceled'" class="uk-button uk-button-default" (click)="getStatus()">Check status</button>

111
utils/dataTransfer/transferData.component.ts
View File

@ -35,7 +35,7 @@ declare var UIkit;
}) })
export class EGIDataTransferComponent { export class EGIDataTransferComponent {
subscriptions = []; subscriptions = [];
statusSub: Subscription = null; statusSub: Subscription[] = [];
accessToken = null; accessToken = null;
@Input() dois; @Input() dois;
loginURL = properties.eoscDataTransferLoginUrl; loginURL = properties.eoscDataTransferLoginUrl;
@ -47,16 +47,17 @@ export class EGIDataTransferComponent {
destinationPath = ""; destinationPath = "";
destinationOptions = null;//properties.eoscDataTransferDestinations.map(dest => {return {"label": dest.destination, "value": dest}}); destinationOptions = null;//properties.eoscDataTransferDestinations.map(dest => {return {"label": dest.destination, "value": dest}});
selectedDestination:{ kind: string, destination: string, description: string, authType: 'token' | 'password' | 'keys', selectedDestination:{ kind: string, destination: string, description: string, authType: 'token' | 'password' | 'keys',
canBrowse: boolean, transferWith: string} = null; protocol: string, canBrowse: boolean, transferWith: string} = null;
folders = {}; folders = {};
files = {}; files = {};
requests: number = 0;
downloadElements = null; downloadElements = null;
@Input() isOpen = false; @Input() isOpen = false;
// @Input() selectedDestinationId = "dcache"; // @Input() selectedDestinationId = "dcache";
@ViewChild('egiTransferModal') egiTransferModal; @ViewChild('egiTransferModal') egiTransferModal;
APIURL = properties.eoscDataTransferAPI; APIURL = properties.eoscDataTransferAPI;
// status: "loading" | "success" | "errorParser" | "errorUser" | "errorTransfer" | "init" | "canceled" = "init"; // status: "loading" | "success" | "errorParser" | "errorUser" | "errorTransfer" | "init" | "canceled" = "init";
status: "unused" | "active" | "succeeded" | "failed" | "canceled" | "errorParser" | "errorUser" | "init" = "init"; status: "unused" | "staging" | "submitted" | "active" | "succeeded" | "partial" | "failed" | "canceled" | "errorParser" | "errorUser" | "init" = "init";
// unused("unused"), // unused("unused"),
// active("active"), // active("active"),
@ -98,9 +99,11 @@ export class EGIDataTransferComponent {
subscription.unsubscribe(); subscription.unsubscribe();
} }
}); });
if(this.statusSub && this.statusSub instanceof Subscriber) { this.statusSub.forEach(sub => {
this.statusSub.unsubscribe(); if(sub instanceof Subscriber) {
} sub.unsubscribe();
}
});
} }
open(){ open(){
@ -143,7 +146,7 @@ export class EGIDataTransferComponent {
this.egiTransferModal.cancelButton = false; this.egiTransferModal.cancelButton = false;
this.egiTransferModal.okButton = true; this.egiTransferModal.okButton = true;
this.egiTransferModal.okButtonText = ">> Transfer"; this.egiTransferModal.okButtonText = ">> Transfer";
this.egiTransferModal.alertTitle = "EOSC data transfer [demo]"; this.egiTransferModal.alertTitle = "EOSC Data Transfer [demo]";
this.egiTransferModal.stayOpen = true; this.egiTransferModal.stayOpen = true;
this.init(); this.init();
if(typeof document !== 'undefined') { if(typeof document !== 'undefined') {
@ -171,6 +174,14 @@ export class EGIDataTransferComponent {
this.jobId = null; this.jobId = null;
// this.statusMessage = null; // this.statusMessage = null;
this.statusMessage = "primary"; this.statusMessage = "primary";
this.requests = 0;
this.folders = {};
this.files = {};
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
} }
checkin(){ checkin(){
window.location.href = this.loginURL+"?redirect="+ encodeURIComponent(window.location.href + (window.location.href.indexOf("&egiTransfer=t")!=-1?"":"&egiTransfer=t")); window.location.href = this.loginURL+"?redirect="+ encodeURIComponent(window.location.href + (window.location.href.indexOf("&egiTransfer=t")!=-1?"":"&egiTransfer=t"));
@ -230,7 +241,7 @@ export class EGIDataTransferComponent {
for (let element of this.downloadElements) { for (let element of this.downloadElements) {
let file = { let file = {
"sources": [element['downloadUrl']], "sources": [element['downloadUrl']],
"destinations": [(this.selectedDestination.destination == "s3" ? "s3://" : "https://") + this.destinationUrl + this.destinationPath + (this.destinationPath.endsWith("/") ? "" : "/") + element.name], "destinations": [(this.selectedDestination.protocol+"://") + this.destinationUrl + this.destinationPath + (this.destinationPath.endsWith("/") ? "" : "/") + element.name],
}; };
//TODO priority? checksum? filesize? //TODO priority? checksum? filesize?
@ -315,7 +326,12 @@ export class EGIDataTransferComponent {
getStatus(updateTransferMessage:boolean = false){ getStatus(updateTransferMessage:boolean = false){
if(this.jobId){ if(this.jobId){
this.requests = 10;
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken}); let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
let source = this.http.get(this.APIURL + "/transfer/" +this.jobId , {headers: headers}).pipe(delay(5000)); let source = this.http.get(this.APIURL + "/transfer/" +this.jobId , {headers: headers}).pipe(delay(5000));
@ -326,9 +342,8 @@ export class EGIDataTransferComponent {
switchMap(() => this.http.get(this.APIURL + "/transfer/" +this.jobId , {headers: headers})) switchMap(() => this.http.get(this.APIURL + "/transfer/" +this.jobId , {headers: headers}))
); );
// this.subscriptions.push(source.pipe(repeat(3)).subscribe( this.statusSub.push(source2.subscribe((res: any) => {
this.statusSub = source2.subscribe( this.requests--;
(res: any) => {
if(this.status != res.jobState) { if(this.status != res.jobState) {
this.status = res.jobState; this.status = res.jobState;
this.jobStatus = res; this.jobStatus = res;
@ -351,12 +366,29 @@ export class EGIDataTransferComponent {
this.message += ` this.message += `
</div>`; </div>`;
this.message += "<div>Transfer status: <b>"+res.jobState+"</b>.</div>";
this.statusMessage = "primary"; this.statusMessage = "primary";
this.statusMessage = res['jobState'] + (res['reason'] ? (" :" + res['reason']) : ""); //this.statusMessage = res['jobState'] + (res['reason'] ? (" :" + res['reason']) : "");
if(this.status == "succeeded") { if(this.status == "partial") {
this.message = "At least one of the selected files was successfully transfered.";
this.statusMessage = "success";
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
} else if(this.status == "succeeded") {
this.message = "Transfer successfully completed!"; this.message = "Transfer successfully completed!";
this.statusMessage = "success"; this.statusMessage = "success";
this.statusSub.unsubscribe(); //this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification('Transfer successfully completed! ', { // UIkit.notification('Transfer successfully completed! ', {
// status: 'success', // status: 'success',
// timeout: 6000, // timeout: 6000,
@ -365,16 +397,28 @@ export class EGIDataTransferComponent {
} else if(this.status == "failed") { } else if(this.status == "failed") {
this.message = "Transfer failed."; this.message = "Transfer failed.";
this.statusMessage = "danger"; this.statusMessage = "danger";
this.statusSub.unsubscribe(); //this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification('Transfer failed', { // UIkit.notification('Transfer failed', {
// status: 'danger', // status: 'danger',
// timeout: 6000, // timeout: 6000,
// pos: 'bottom-right' // pos: 'bottom-right'
// }); // });
} else if(this.status != "active") { } else if(this.requests <= 0 || (this.status != "active" && this.status != "submitted" && this.status != "staging")) {
this.message = "Transfer completed with status: <b>"+this.status+"</b>."; this.message = "Transfer status: <b>"+this.status+"</b>.";
this.statusMessage = "warning"; this.statusMessage = "warning";
this.statusSub.unsubscribe(); //this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification('Transfer completed with status: '+this.status, { // UIkit.notification('Transfer completed with status: '+this.status, {
// status: 'warning', // status: 'warning',
// timeout: 6000, // timeout: 6000,
@ -382,11 +426,26 @@ export class EGIDataTransferComponent {
// }); // });
} }
} }
if(this.requests <= 0) {
this.message = "Transfer status: <b>"+this.status+"</b>.";
this.statusMessage = "warning";
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
}
}, error => { }, error => {
this.status = "failed"; this.status = "failed";
this.message = "Status of the transfer could not be retrieved."; this.message = "Status of the transfer could not be retrieved.";
this.statusMessage = "danger"; this.statusMessage = "danger";
this.statusSub.unsubscribe(); this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification("Couldn't get status", { // UIkit.notification("Couldn't get status", {
// status: 'error', // status: 'error',
// timeout: 6000, // timeout: 6000,
@ -394,7 +453,7 @@ export class EGIDataTransferComponent {
// }); // });
} }
); ));
} }
} }
@ -414,7 +473,7 @@ export class EGIDataTransferComponent {
} }
hasBrowse(){ hasBrowse(){
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken}); let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
this.subscriptions.push(this.http.get(this.APIURL + "/storage/info?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent(this.destinationUrl + this.destinationPath) , {headers: headers}).subscribe( this.subscriptions.push(this.http.get(this.APIURL + "/storage/info?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent((this.selectedDestination.protocol+'://') + this.destinationUrl + this.destinationPath) , {headers: headers}).subscribe(
res => { res => {
console.log(res); console.log(res);
} }
@ -424,7 +483,7 @@ export class EGIDataTransferComponent {
getFolder(folderPath){ getFolder(folderPath){
//TODO is this necessary? //TODO is this necessary?
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken}); let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent(this.destinationUrl + folderPath) , {headers: headers}).subscribe( this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent((this.selectedDestination.protocol + '://') + this.destinationUrl + folderPath) , {headers: headers}).subscribe(
res => { res => {
this.folders[folderPath]= res; this.folders[folderPath]= res;
this.folders[folderPath]['isOpen'] = true; this.folders[folderPath]['isOpen'] = true;
@ -439,7 +498,7 @@ export class EGIDataTransferComponent {
} }
this.getFolder(folderPath); this.getFolder(folderPath);
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken}); let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder/list?dest="+this.selectedDestination.destination+"&folderUrl="+encodeURIComponent(this.destinationUrl + folderPath) , {headers: headers}).subscribe( this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder/list?dest="+this.selectedDestination.destination+"&folderUrl="+encodeURIComponent((this.selectedDestination.protocol + '://') + this.destinationUrl + folderPath) , {headers: headers}).subscribe(
res => { res => {
this.files[folderPath]= res['elements']; this.files[folderPath]= res['elements'];
} }
@ -473,13 +532,13 @@ export class EGIDataTransferComponent {
pathValidator(): ValidatorFn { pathValidator(): ValidatorFn {
return (control: AbstractControl): { [key: string]: string } | null => { return (control: AbstractControl): { [key: string]: string } | null => {
if (!this.validatePath()) { if (!this.validatePath()) {
return {'error': 'Path should start with "/" e.g /path'}; return {'error': 'Path should start with "/"'};
} }
return null; return null;
} }
} }
validatePath():boolean { validatePath():boolean {
let exp1 = /^\/([A-z0-9-_+])*$/g; let exp1 = /^\/([A-z0-9-_+]+\/?)*$/g;
return (this.destinationPath.length > 0 && this.destinationPath.match(exp1) != null) return (this.destinationPath.length > 0 && this.destinationPath.match(exp1) != null)
} }