MaDgIK
/
openaire-library
13
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data-transfer-v2 #6

Merged
konstantina.galouni merged 26 commits from data-transfer-v2 into develop 2023-06-13 16:20:04 +02:00
Conversation 0 Commits 26 Files Changed 6 +93 -36
2 changed files with 93 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit aa5425d091 - Show all commits

18
utils/dataTransfer/transferData.component.html
View File

@ -10,8 +10,8 @@
<!-- This is the modal -->
<modal-alert #egiTransferModal large="true" [okDisabled]="destinationPath.length == 0 || status == 'succeeded'
||status == 'active' || !validatePath() || !validateDestinationUrl() || (!this.downloadElements || this.downloadElements.length == 0)"
(alertOutput)="transfer()" >
|| (requests > 0) || !validatePath() || !validateDestinationUrl() || (!this.downloadElements || this.downloadElements.length == 0)"
(alertOutput)="transfer()" (cancelOutput)="init()">
<div *ngIf="!accessToken" class="">
<div class="uk-width-1-1 uk-margin-top uk-margin-bottom uk-text-center">
In order to send data to a Cloud Storage, you would need to be authenticated, please login via EGI check-in.
@ -51,19 +51,17 @@
<!-- /Demonstrators/EOSC-Future/EGI/-->
<!-- <br>-->
<!-- -->
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Please select the
Destination Storage type:</p>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Destination storage type:</p>
<div input type="select" [(value)]="selectedDestination" hint="Select..."
[options]="destinationOptions" (valueChange)="folders = {}"></div>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Specify the destination system (e.g. hostname:8080):</p>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Destination system (e.g. hostname:8080):</p>
<div input [(value)]="destinationUrl"
[validators]="URLValidators" class=""></div>
<ng-container
*ngIf="selectedDestination.authType == 'password' || selectedDestination.authType == 'keys'">
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Provide
authentication if needed:</p>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Authentication:</p>
<div class="uk-grid uk-child-width-1-2">
<div input [(value)]="destinationAuthUser" [placeholder]="'Give ' + (selectedDestination.authType ==
'password'? 'username':'access key') "
@ -74,7 +72,7 @@
</div>
</ng-container>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Specify the destination path (e.g. /folder1/folder2):</p>
<p class="uk-text-meta uk-text-xsmall uk-margin-remove-bottom uk-margin-top">Destination path (e.g. /folder1/folder2):</p>
<div input [(value)]="destinationPath"
[validators]="pathValidators" class=""></div>
<!-- <div *ngIf="selectedDestination.hasBrowse">-->
@ -100,7 +98,7 @@
<!-- <a *ngIf="status != 'succeeded' && status != 'active'" (click)="transfer()">-->
<!-- Try again!-->
<!-- </a>-->
<div *ngIf="status == 'active'" class="uk-flex uk-flex-center uk-text-muted">
<div *ngIf="requests > 0" class="uk-flex uk-flex-center uk-text-muted">
<div>
<span class="uk-icon uk-spinner">
<svg width="60" height="60" viewBox="0 0 30 30" xmlns="http://www.w3.org/2000/svg"
@ -109,7 +107,7 @@
</span>
</div>
</div>
<a *ngIf="status != 'active'" class="uk-alert-close" uk-close></a>
<a *ngIf="requests <= 0" class="uk-alert-close" uk-close></a>
</div>
<!-- <div *ngIf="jobId || status == 'canceled'">
<button *ngIf=" status != 'canceled'" class="uk-button uk-button-default" (click)="getStatus()">Check status</button>

111
utils/dataTransfer/transferData.component.ts
View File

@ -35,7 +35,7 @@ declare var UIkit;
})
export class EGIDataTransferComponent {
subscriptions = [];
statusSub: Subscription = null;
statusSub: Subscription[] = [];
accessToken = null;
@Input() dois;
loginURL = properties.eoscDataTransferLoginUrl;
@ -47,16 +47,17 @@ export class EGIDataTransferComponent {
destinationPath = "";
destinationOptions = null;//properties.eoscDataTransferDestinations.map(dest => {return {"label": dest.destination, "value": dest}});
selectedDestination:{ kind: string, destination: string, description: string, authType: 'token' | 'password' | 'keys',
canBrowse: boolean, transferWith: string} = null;
protocol: string, canBrowse: boolean, transferWith: string} = null;
folders = {};
files = {};
requests: number = 0;
downloadElements = null;
@Input() isOpen = false;
// @Input() selectedDestinationId = "dcache";
@ViewChild('egiTransferModal') egiTransferModal;
APIURL = properties.eoscDataTransferAPI;
// status: "loading" | "success" | "errorParser" | "errorUser" | "errorTransfer" | "init" | "canceled" = "init";
status: "unused" | "active" | "succeeded" | "failed" | "canceled" | "errorParser" | "errorUser" | "init" = "init";
status: "unused" | "staging" | "submitted" | "active" | "succeeded" | "partial" | "failed" | "canceled" | "errorParser" | "errorUser" | "init" = "init";
// unused("unused"),
// active("active"),
@ -98,9 +99,11 @@ export class EGIDataTransferComponent {
subscription.unsubscribe();
}
});
if(this.statusSub && this.statusSub instanceof Subscriber) {
this.statusSub.unsubscribe();
}
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
}
open(){
@ -143,7 +146,7 @@ export class EGIDataTransferComponent {
this.egiTransferModal.cancelButton = false;
this.egiTransferModal.okButton = true;
this.egiTransferModal.okButtonText = ">> Transfer";
this.egiTransferModal.alertTitle = "EOSC data transfer [demo]";
this.egiTransferModal.alertTitle = "EOSC Data Transfer [demo]";
this.egiTransferModal.stayOpen = true;
this.init();
if(typeof document !== 'undefined') {
@ -171,6 +174,14 @@ export class EGIDataTransferComponent {
this.jobId = null;
// this.statusMessage = null;
this.statusMessage = "primary";
this.requests = 0;
this.folders = {};
this.files = {};
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
}
checkin(){
window.location.href = this.loginURL+"?redirect="+ encodeURIComponent(window.location.href + (window.location.href.indexOf("&egiTransfer=t")!=-1?"":"&egiTransfer=t"));
@ -230,7 +241,7 @@ export class EGIDataTransferComponent {
for (let element of this.downloadElements) {
let file = {
"sources": [element['downloadUrl']],
"destinations": [(this.selectedDestination.destination == "s3" ? "s3://" : "https://") + this.destinationUrl + this.destinationPath + (this.destinationPath.endsWith("/") ? "" : "/") + element.name],
"destinations": [(this.selectedDestination.protocol+"://") + this.destinationUrl + this.destinationPath + (this.destinationPath.endsWith("/") ? "" : "/") + element.name],
};
//TODO priority? checksum? filesize?
@ -315,7 +326,12 @@ export class EGIDataTransferComponent {
getStatus(updateTransferMessage:boolean = false){
if(this.jobId){
this.requests = 10;
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
let source = this.http.get(this.APIURL + "/transfer/" +this.jobId , {headers: headers}).pipe(delay(5000));
@ -326,9 +342,8 @@ export class EGIDataTransferComponent {
switchMap(() => this.http.get(this.APIURL + "/transfer/" +this.jobId , {headers: headers}))
);
// this.subscriptions.push(source.pipe(repeat(3)).subscribe(
this.statusSub = source2.subscribe(
(res: any) => {
this.statusSub.push(source2.subscribe((res: any) => {
this.requests--;
if(this.status != res.jobState) {
this.status = res.jobState;
this.jobStatus = res;
@ -351,12 +366,29 @@ export class EGIDataTransferComponent {
this.message += `
</div>`;
this.message += "<div>Transfer status: <b>"+res.jobState+"</b>.</div>";
this.statusMessage = "primary";
this.statusMessage = res['jobState'] + (res['reason'] ? (" :" + res['reason']) : "");
if(this.status == "succeeded") {
//this.statusMessage = res['jobState'] + (res['reason'] ? (" :" + res['reason']) : "");
if(this.status == "partial") {
this.message = "At least one of the selected files was successfully transfered.";
this.statusMessage = "success";
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
} else if(this.status == "succeeded") {
this.message = "Transfer successfully completed!";
this.statusMessage = "success";
this.statusSub.unsubscribe();
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification('Transfer successfully completed! ', {
// status: 'success',
// timeout: 6000,
@ -365,16 +397,28 @@ export class EGIDataTransferComponent {
} else if(this.status == "failed") {
this.message = "Transfer failed.";
this.statusMessage = "danger";
this.statusSub.unsubscribe();
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification('Transfer failed', {
// status: 'danger',
// timeout: 6000,
// pos: 'bottom-right'
// });
} else if(this.status != "active") {
this.message = "Transfer completed with status: <b>"+this.status+"</b>.";
} else if(this.requests <= 0 || (this.status != "active" && this.status != "submitted" && this.status != "staging")) {
this.message = "Transfer status: <b>"+this.status+"</b>.";
this.statusMessage = "warning";
this.statusSub.unsubscribe();
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification('Transfer completed with status: '+this.status, {
// status: 'warning',
// timeout: 6000,
@ -382,11 +426,26 @@ export class EGIDataTransferComponent {
// });
}
}
if(this.requests <= 0) {
this.message = "Transfer status: <b>"+this.status+"</b>.";
this.statusMessage = "warning";
//this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
}
}, error => {
this.status = "failed";
this.message = "Status of the transfer could not be retrieved.";
this.statusMessage = "danger";
this.statusSub.unsubscribe();
this.statusSub.forEach(sub => {
if(sub instanceof Subscriber) {
sub.unsubscribe();
}
});
this.requests = 0;
// UIkit.notification("Couldn't get status", {
// status: 'error',
// timeout: 6000,
@ -394,7 +453,7 @@ export class EGIDataTransferComponent {
// });
}
);
));
}
}
@ -414,7 +473,7 @@ export class EGIDataTransferComponent {
}
hasBrowse(){
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
this.subscriptions.push(this.http.get(this.APIURL + "/storage/info?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent(this.destinationUrl + this.destinationPath) , {headers: headers}).subscribe(
this.subscriptions.push(this.http.get(this.APIURL + "/storage/info?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent((this.selectedDestination.protocol+'://') + this.destinationUrl + this.destinationPath) , {headers: headers}).subscribe(
res => {
console.log(res);
}
@ -424,7 +483,7 @@ export class EGIDataTransferComponent {
getFolder(folderPath){
//TODO is this necessary?
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent(this.destinationUrl + folderPath) , {headers: headers}).subscribe(
this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder?dest="+this.selectedDestination.destination+"&seUrl="+encodeURIComponent((this.selectedDestination.protocol + '://') + this.destinationUrl + folderPath) , {headers: headers}).subscribe(
res => {
this.folders[folderPath]= res;
this.folders[folderPath]['isOpen'] = true;
@ -439,7 +498,7 @@ export class EGIDataTransferComponent {
}
this.getFolder(folderPath);
let headers = new HttpHeaders({'Authorization': 'Bearer '+this.accessToken});
this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder/list?dest="+this.selectedDestination.destination+"&folderUrl="+encodeURIComponent(this.destinationUrl + folderPath) , {headers: headers}).subscribe(
this.subscriptions.push(this.http.get(this.APIURL + "/storage/folder/list?dest="+this.selectedDestination.destination+"&folderUrl="+encodeURIComponent((this.selectedDestination.protocol + '://') + this.destinationUrl + folderPath) , {headers: headers}).subscribe(
res => {
this.files[folderPath]= res['elements'];
}
@ -473,13 +532,13 @@ export class EGIDataTransferComponent {
pathValidator(): ValidatorFn {
return (control: AbstractControl): { [key: string]: string } | null => {
if (!this.validatePath()) {
return {'error': 'Path should start with "/" e.g /path'};
return {'error': 'Path should start with "/"'};
}
return null;
}
}
validatePath():boolean {
let exp1 = /^\/([A-z0-9-_+])*$/g;
let exp1 = /^\/([A-z0-9-_+]+\/?)*$/g;
return (this.destinationPath.length > 0 && this.destinationPath.match(exp1) != null)
}