From a6411ba7ea104ab8dc4552c1274c17bcd8248052 Mon Sep 17 00:00:00 2001 From: giancarlo Date: Wed, 7 Oct 2020 09:53:01 +0200 Subject: [PATCH] First Release --- .gitignore | 4 + .project | 17 +++ .pydevproject | 8 ++ .settings/org.eclipse.core.resources.prefs | 13 +++ .../org.eclipse.ltk.core.refactoring.prefs | 2 + README.md | 11 ++ __main__.py | 4 + dockerimageexecutor.py | 98 ++++++++++++++++ dockerimageexecutor.zip | Bin 0 -> 10762 bytes engine/__init__.py | 1 + engine/dmonitor.py | 33 ++++++ engine/dockerengine.py | 73 ++++++++++++ engine/swarmengine.py | 108 ++++++++++++++++++ issupport.py | 47 ++++++++ storagehub/__init__.py | 1 + storagehub/storagehubcommand.py | 17 +++ storagehub/storagehubcommandcreatefolder.py | 45 ++++++++ .../storagehubcommandcreatetempfolder.py | 45 ++++++++ storagehub/storagehubcommanditemdelete.py | 34 ++++++ storagehub/storagehubcommanditemdownload.py | 36 ++++++ storagehub/storagehubcommanditemupload.py | 45 ++++++++ storagehub/storagehubcommandrootitemid.py | 35 ++++++ 22 files changed, 677 insertions(+) create mode 100644 .gitignore create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 .settings/org.eclipse.ltk.core.refactoring.prefs create mode 100644 README.md create mode 100644 __main__.py create mode 100644 dockerimageexecutor.py create mode 100644 dockerimageexecutor.zip create mode 100644 engine/__init__.py create mode 100644 engine/dmonitor.py create mode 100644 engine/dockerengine.py create mode 100644 engine/swarmengine.py create mode 100644 issupport.py create mode 100644 storagehub/__init__.py create mode 100644 storagehub/storagehubcommand.py create mode 100644 storagehub/storagehubcommandcreatefolder.py create mode 100644 storagehub/storagehubcommandcreatetempfolder.py create mode 100644 storagehub/storagehubcommanditemdelete.py create mode 100644 storagehub/storagehubcommanditemdownload.py create mode 100644 storagehub/storagehubcommanditemupload.py create mode 100644 storagehub/storagehubcommandrootitemid.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4765f1e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/globalvariables.csv +/sortableelemnts.txt +/sortapp-img.tar.gz +/result.zip diff --git a/.project b/.project new file mode 100644 index 0000000..5bfebbd --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + dockerimageexecutor + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..70098b5 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + +python interpreter +Default + +/${PROJECT_DIR_NAME} + + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..3a45744 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,13 @@ +eclipse.preferences.version=1 +encoding//engine/dmonitor.py=utf-8 +encoding//engine/dockerengine.py=utf-8 +encoding//engine/swarmengine.py=utf-8 +encoding//storagehub/storagehubcommand.py=utf-8 +encoding//storagehub/storagehubcommandcreatefolder.py=utf-8 +encoding//storagehub/storagehubcommandcreatetempfolder.py=utf-8 +encoding//storagehub/storagehubcommanditemdelete.py=utf-8 +encoding//storagehub/storagehubcommanditemdownload.py=utf-8 +encoding//storagehub/storagehubcommanditemupload.py=utf-8 +encoding//storagehub/storagehubcommandrootitemid.py=utf-8 +encoding/dockerimageexecutor.py=utf-8 +encoding/issupport.py=utf-8 diff --git a/.settings/org.eclipse.ltk.core.refactoring.prefs b/.settings/org.eclipse.ltk.core.refactoring.prefs new file mode 100644 index 0000000..b196c64 --- /dev/null +++ b/.settings/org.eclipse.ltk.core.refactoring.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false diff --git a/README.md b/README.md new file mode 100644 index 0000000..0b9db81 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +#Docker Image Executor + +This is a simple algorithm that execute image on Docker using a image file or Swarm Cluster using local registry. + +- python3 dockerimageexecutor.py + + +##Example + +- python3 dockerimageexecutor.py "microservices-VirtualBox:443/sortapp" sortapp 548eade8-25cf-4978-9f61-0f0c652900be + diff --git a/__main__.py b/__main__.py new file mode 100644 index 0000000..5ecce1d --- /dev/null +++ b/__main__.py @@ -0,0 +1,4 @@ +print('dockerimageexecutor/__main__.py executed') +#import dockerimageexecutor + + diff --git a/dockerimageexecutor.py b/dockerimageexecutor.py new file mode 100644 index 0000000..28622d2 --- /dev/null +++ b/dockerimageexecutor.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2020/06/12 +# +import os +import sys + +from issupport import ISSupport +from engine.dockerengine import DockerEngine +from engine.swarmengine import SwarmEngine +from storagehub.storagehubcommanditemdownload import StorageHubCommandItemDownload +from storagehub.storagehubcommandcreatetempfolder import StorageHubCommandCreateTempFolder +from storagehub.storagehubcommanditemdelete import StorageHubCommandItemDelete + + +class DockerImageExecutor: + + def __init__(self): + self.resultFile = "result.zip" + self.globalVariablesFile = "globalvariables.csv" + self.gcubeToken = None + self.storageHubUrl = None + self.softwareImage = sys.argv[1] # Software Image + self.softwareExecuteCommandName = sys.argv[2] # Command to Run + self.fileItemId = sys.argv[3] # Input Data File + self.tempFolderItemId = None # '32c0422f-a777-4452-adea-007347ec4484' + + def main(self): + print(self) + self.retrieveToken() + issup = ISSupport() + self.storageHubUrl = issup.discoverStorageHub(self.gcubeToken) + self.createTempFolder() + # self.executeOnDocker() + self.executeOnSwarm() + + def retrieveToken(self): + print("Retrieve gcubeToken") + if not os.path.isfile(self.globalVariablesFile): + print("File does not exist: " + self.globalVariablesFile) + raise Exception("File does not exist: " + self.globalVariablesFile) + with open(self.globalVariablesFile) as fp: + for line in fp: + if line.find("gcube_token") != -1: + tk = line[14:] + self.gcubeToken = tk.replace('"', '').strip() + print("Found gcube_token") + break + if self.gcubeToken == None: + print('Error gcube_token not found!') + raise Exception('Error gcube_token not found!') + + def createTempFolder(self): + print("Create Temp Folder") + cmdCreateTempFolder = StorageHubCommandCreateTempFolder(self.gcubeToken, self.storageHubUrl) + self.tempFolderItemId = cmdCreateTempFolder.execute() + + def deleteTempFolder(self): + print("Delete Temp Folder") + cmdDeleteTempFolder = StorageHubCommandItemDelete(self.gcubeToken, self.storageHubUrl, self.tempFolderItemId) + cmdDeleteTempFolder.execute() + + def downloadResults(self): + print("Get Results") + cmdItemDownload = StorageHubCommandItemDownload(self.gcubeToken, self.storageHubUrl, + self.tempFolderItemId, self.resultFile) + cmdItemDownload.execute() + self.deleteTempFolder() + + def executeOnDocker(self): + print("Execute On Docker") + dEngine = DockerEngine(self.gcubeToken, self.storageHubUrl, + self.softwareImage, self.softwareExecuteCommandName, self.fileItemId, self.tempFolderItemId) + dEngine.execute() + self.downloadResults() + + def executeOnSwarm(self): + print("Execute On Swarm") + sEngine = SwarmEngine(self.gcubeToken, self.storageHubUrl, + self.softwareImage, self.softwareExecuteCommandName, self.fileItemId, self.tempFolderItemId) + sEngine.execute() + self.downloadResults() + + def __str__(self): + return 'DockerImageExecutor' + + +def main(): + print('docker_image_executor') + dockerImageExecutor = DockerImageExecutor() + dockerImageExecutor.main() + + +main() + diff --git a/dockerimageexecutor.zip b/dockerimageexecutor.zip new file mode 100644 index 0000000000000000000000000000000000000000..476522493eee0e7b0840551cad83fa4ffe09c271 GIT binary patch literal 10762 zcma)i1yq&W*7l~mQ@W+QyQE7RM3K(Trn^HrMUd|9?hX-<7LiU70cmOYH|OXPKF|H{ z-DAAt-E)jJ=Ck5?pEc)POGO?E8XF1#fCsR7kw1-Aq!-u$002%90008`t@=bvQt=5h z&^S_AOsSU@xpfm~`bTQsPQ85i7>TctEvy|+(VP^O;zr}xO}e!Pd6Z1MtEnKzNP56I z)hoPc;w}>!4%wG^i|&NvDB`u=j?iD5J2lHK@i*-bhBAd^ZV1ylI*SC zlpe13YVrD!NU+4LPTYtr9(kM*;l~EN;x+DNxV2of(MW>5V{-NsZK|6hotn}q%R;D; zBxCyL@fp41FXp}=%lQpf4@GIdpff(xa)XtXkU(Z_w~U5exsDXVSCDCtr6c1UIt!N000+^Vhl2}GO@P+8km`w zxSAL_IfCrLfFCs#qbFI>+cve37bJAeCvE7k%hVANaKFN!Bk7?R&l^wZD((z855B8- zT>qMT1n&8=q*CZoO4a5EUJfm^{3h&4CKEMd=>hLTA+tps1rjB)bqtdh!T}9Iw|uSE z5=b|tLw`z}WiDz)DvFi;bXJ1zMSlA5REU(z$X{yCt@%Cp+tMWZH4G(_!?MAx&#EU^SHW)rVK?#oQ@by;LMP@bS?%sL+Fg zPb_Q zLvvHQj5Lgi6g?{^PH@qy1iYJW5VdTzvNKmmn)=bM6;_+!wN$tNhhg$gS zo;a!VcwFHiwr~v{X1j+{WK!Fqe55;?Zbep}JCEL6v1ZG7hgGe|{5ZVt^?9JA3{B1F z!vn+66~hDTm5li``y|Z4LBaG+1(p0ALykH*6+zvT;KyT&YT9WP1zi`f4N?3<24hH{ zrpnvT4A}N4&J^Cxa%&~{adYgwWi|+2?*20PZ0chVN)^4WR#I1R9Q5XER7=a=&PGV) zfQUjb(Y%q<%8IWH3k$@YkQmYydMhsUA5oB(F^-$2I{BO=HX&=dUNQAK&5ZrGWqdBhgNnX;4d zlup;h1%YY;hOV_(iKkaZ0xv_3TVQQh2vaLnNWM%w@*;)Y1f|84GK>_WStsrM?EGwR zO3T5AXD_i)i+HgR*%Q3qt+rm!JQC`_>wtG5ddw6XzhCMnQ~;CZigdEZdvc4sS-FHI zH|4!2ulI!0p@;=u zeogu^5tI_BYx=48xhB5@g>$?W%xH}GzJI(cQNK;RofV!@%t9B5< @*?MbjH%0}H zjBmLM5EZfF6!hMolP{fnfwGD`uyZd)VX_8YMjeG~AZE**14#JmEP~ANHw^ceU|Q z{OwlzT1F&2zAKVxIuH@K3*@_jkZt{xtad3ERFd6u|o*Z_gwfxA-n=CrZ=Pa-}69bB58R8k-wP9okF2 zz1X6+*k`9t5gu z=euc(lnA;@f2boi24hqeMp68hZFKICnGkoC3SdFq!$c>TX&S zF+sM-Hu6x3gGkbiwM-Oz1;W#)>CER+=nEuobdPG6SC@n%8)+w z@8uwViF|#IL%~V_M-$D{v>vf5T}*x1$DG0Yte45Xevw&po#06+!}0KoaJ+zu`V_P~F>^gJz{yO_>{?#%;Ous|}83 zwoeIk1yE|w>xnWW_BY6zADbSG$@o)F36i52u4BGn>vo`!huA|fJCCaSW)_F3h=fk0 z6r~(Bxpuj=izWJ`vshOKC1}7$wMqY4#edJoN^|@r<&zE8(Du?IIqWxPj0K29J5UFZQYfOPy9iT3glTc$XIWYn*C0a?eafu%Z%G|1Opd63rPEv2RAco zV~pkXd6q=pVTdeIbz?XMCSsWcc+Kex~=3*%eXx>k*cd83I5 z-ULqBgt7K8tD|m(3*W)mwVjZBLm}-d%7Ggm3yO>pWs%WHeCwyvXHWN|=$WUfDo?H_ zJ#&J~-l8ndm)NNje~mev3`rI!nd}DP^|=jLJ$pb6NA7AXilE|Syago&G-|`XZK)oI zY=qWcqpiyWxFF~2Y&3#&GV$Q-<-sit0}X!;rgiCT@8((6L&g^`lFF&q_>w{d9ZcYCamBXB zsLH5iO*utu0$;<;SwJ@s6D@`su9`mab8kVcY~i ze`r9*QVLS}qjyKdA}=0s!?;u_3Qg(5_bfa!N2ikv4sc8D`KFXYUeFl$80@;NQLlpmVD|N$2(UcB$KG%4QdiYcro>b~oE#fBY z#C2|B8U1(daOni0ARG4VJ;`w=O7O%OuWEfcH9Gbe;n2K+{HxlE1@t9XavW99Fu3KBudHOqCf?k_aNy-12|kLzipU=&ce9#`tAf}m zQWP#|2SKMjL!o^1Dy`ux;C!@f&U(*U_?y^-(&2V9@kY(!OHu`#&v%+#!LT% zb#{Sti6_j{k|~XK?As3g2dbk{1!yA~gs<380yI`)og?@F-K5hBn0g(~pneBFL)8;1M7W=F%fPYb4J-hF z3|2!62L~rxTadjYINV9oSOWbeg8FttB(so;{Ltpb*buXWGf2r_U6b<%otTTBSO(Xq zIo%2+mO6N2he_rZM|SMBw_JKUkB+W_6GBaXskVT__rM6$5rYd$#ED$FO@~4~?zU1b ze+Mby&@evEU_$CzY<2ubEZz5!mdgy{E37a%E4w?Z?CD-1EQ?T3ezOU$h|q6}$M^}2 zjq59ya54DB1EE@JZNq05j#Sa1tI?;sckOlFaPbF|0lZa*U)HJWHhr|Tt{R|;=v?75 zbN%F2axg%#8{bQ+E(*EQ-!w6@rk#S#Vmj%V++Ks>lPk`|#vUI`>=&o-P5(B#-!QPL zIw~DjJV3@wX8o*AmnDO%JQ-DILznkJN3QmVKKW8F?siBX+$=muxww>5AU9zO{{+@v zr|e+yN>V84vNA5-d2Rm!9h!J6^s$CIBQchPOq8`Nia{xrK%Gxm3i1I&1gIQ&MZ6$y z1>0_5SWT*OB!*)rm5ww8TTRO)mbGm%z9NC_pays2Yk&LI`%K`fWVeQMA5`?a%$_zY zEe%*9;Z$nDuN)wgPa`m~5gk-r*-PqGfPALjM}x=Y8e8W6V>8B=Q(Sja+NR4$-}C(NY6x zrFGEqd4a8UU-KJaL8i1TJYKT0w@~^DC6y;P%EnWU_^0%#A(E3{heScePE&jaZI-V% z)`hMO>C@bil+^c5t zyd^ytTa(1jU zM>;;G#NY;Z&s~Y^Vgv#L4Q!16NZ=a4A5yZ%V(e(-eKuGq znD7z-wdGppyOO<6AbcQtv&Kl*K!hm;mJ@vE3kp0Kj z0~LuJUQJ0;f-dPtp6V-m>=&KTB}uKx!mCKX)uRqPSi0L3$fscSU;yXI4F6@YM)oEK zjwYrcYh#nUs#%%()4#;!w4%`j%|d(5Q6`g)7SdvSh6*~Q>Tw`FPg5;!=yPj09P%3h zrHOt!J#@4q35}&wFPF`hp|p&?y10Yn1N?Kend_-%UGMpZ(?5`5@vWsmTRtByCqUxk z86eapj8a%;$`Q<|#@>&$PXOv5>C5L5IG_`yvoKez_6?mK!r=Q>T0YK&4PCNQ*Tqnk zdD^9CLB0jsaKewJ@x%=kZdC$q{^RyGDp{a0wZ@=yb7n*K@KYs0PPt6JK+X<6+)7=x zkz-4)r`ot`GsY(bBH^Hcf=u^=9L55bTIM}2&C~OPAL*K|S3UvsudNG`mZc|&haA=| zRK`>dUK(QYrqayL!v;8_f2QWs!B2N^De6H+ad34C<#v;(!Ao3k8?+UglQGm^sNVgE zc{#X7Qlo41?2LBnAkA1$+217irlmuSGcK)g1_qjKQoD%P7pX&uDZNg3Z4PbECL!Ym z+&BYdz@$F^b0w}IZ112{O?;ZQFeUO~pcmu!NSQ_;lF_4j?T&Y&6ZVDy(ijJ-B_`*Y zC@xTRg0dL1cBxyYB$F>345g@ZugwIP5b_>-(W{wPD*{*7Y-C4YPUJUhmME;7i)s1> zbm~~*4t9?W+uGz}W8%YIWkASfx~1SvlgK8CYk%jkZuyaW(-P6Mmav08=0xQ;2A|SC z)YRWba;%qKP8~cVtY`enYsTm)q|pV{F(!~g=)0WQ5zBW?+32ck*)U}BmyY6Q9~-}s z+v^I>=ao;--ew=4#%^~8C|}x5*KKxFMDkw|NQvIuE8W(BnOR>$*@gyYW&?O%WBH$$ z>1YD9{fC`->O(ejtmv&r4E2Xystj|od4!{uX+hf!-(W zs@trV2qL^V5ycAK#5k$Du8-O^Wl(6yU_4tJTXpq3b@aSp@yG<1&q-YpZJ%?SxU`ZaG zJkUgan{7(y%O&F0DS8)Xn1sxMB5M%u7ylIAA6%vu@Xt(nTv={&;Ke?zi_kkrLcUWI z-9*96#?mF6it{aG`ur-He_NkZI{3t zkQ3dg=Am&MdJQcf8E;BJs0YL&iO~tjd)8GQwTFY(f7-4wo=CU`lRNZ0=P3~>dRVBN z)JL*}3?CuusIOVV+$Ao3PB6%sohojtpyQ>}GXI#@~@vQ6;jIM1tws z^XQO9u!eeNfBAIouRkeU#DO0-^$uUrgy#zBmv*ois`dF%ul~gXrE+8P5 zR?d7j}8+?ckY4cPM^ic`y3 zew>&+AHvj5u@1|if+@_%!aTWAaJ)*%+bKCm$1Zxdl+B&|F;`dHJSU{W&PXfI@)i&a?`)IHDT1TBM>-fRlyRnf$c` zAJneMjp7I7z?!xq(=l?s$MN}Y13Ae&1eBYDFB{x5`W9gip+;4(&jgSWuJV!fj5+R_LCXGy8C(5p^P%FSlg*y>A%+pji z6ppDpTdz6&!IO~uOrm^`ESP7$_}yN8)374x@C!~%T(R$Sj!#h{H@5k2NZ0At`@eNZ zB7M&)Bqy^?Xz|m&fcM7_!S;*sW!a1s(?8$J_r|~@Sh;BL9f7CT-y=bd=B`ai>v=@R znLwqmX4ZNOE&Iie)Hf^ETaL)Uwsy6^ZFr6K?5Q1Z?SZRAug@Cv4CU`_uoz5;3p2`- zZ(u^mfC=&6w!yy$0dldi1{oONQ9|vXZLkhpXaMq4jj5~qal-DPSnLO?Y|tc5QB(zZ z<*zL{$jT)fj|b+TliOc;ou0QJEnHFj80q)OGn(qix{!d&=_5*DNr-u?zU2v(UcI5= zMe!w~H8Gvf87d=wDzrApk43d)J(J*UHk(SJ%iOiDy*2C;Xm#jQr>}~M38P=1J_#{j ztd3guU~CopyMU#~`HXo=E4?jNexu$#3y|U=xiKR`k~RoKXkQAEC@1$J2!NR{#HW_9 zIc9d_HEZXBP{XYu+*VJ|K3++-0Y1U_)bv4OBQ514McL=pQkfh}YmhM@@|NiuIVJc4 zl|eED<{Y4Xk=*)I_JRfEuBYt71Zj@PBW#OCs|bWWjy8c%!aM4ijWsvRey`?sFDSgrIN%!& z6o*1(4sG5|`kJYju!=4l4qdm~SP7I@)b;yM_};30MVSg!)|=a4ek0M?q+D67f5ms~ zv1hC92vEyHEl37JblH(gk`WtU2~^IdmPTn}*nh;#LEw6FUFBe`toA8`EPTD}6{{=f z?p}cR4D62I22C@op=75y%&ln>RPZi`BFSfA=}3Sus<*9Bul{e!6oM&Jh1EaG2i~6w z!28qxq9IPUzX+rLAqt$7wO`Vi)&x^=0-I$rHDX{UR5LN*a!r)sHEdSFaTD=f+_Xq6 zfNHTO*^+qDRlS)4x)Ds@E~(XEU*Jery+KM|x}fE4ezg%!y`KH@Dn_hiXrt^+rd2>I zW!ZS_BTwVNO-W72cJz@$%I#$V+0c9#cvO~8JPlhHPVPsD$iCVO@I_)18B7|wS*l?K znsP|8sK+zk9iDy)#2edQ5Li}g>2Lsu(^iD2su`D9T#u>yV~<0Jb5tUcx8T(VE=1Yo zyR^uR7b`5sGGq;QU}=yKAd*#14i2WTE|^y4Aj z7ilO^?fdMS1a8oRQ7=o~AEC6jx?0jOTJz}dtoC1b1+*Dgx{$^P_oyhjXS5Uq_9`{q!!z%pBQ z$li~2ZD%{}>FHZPz=Lnaapyn^u7{~jJ(oCOyqvgt>lxY$`?h+tXUyowgiiF$43V19 z*ab>QFk7b2Z~oyoOsIT5QcQUdmaG_XCG@}X&mIJFypt`9zXj_#IBv)G0>|xE)N!4& zcX9gwq($;eL2_7SHJqj&idHf^hWa^&dv+5s+|RiJ14#}0Hd8O{JcZf@0$k4JQiwh% z&)j53a#`8_YX?1!qMPQfL@5|yd@iYg3h!DkLraOoF zK039#rs=o&!?P|}mr>fDQD#2A-MmeSMqn@_eCtuIRBDH%Il&#tis(LaO>%~=$>ZV6 zMq{;dZM0rf=O{fhUv!L-fHTJ2KU>j_IH*dfXp-b!S|YDKRN2k)Lm*jxKj5sSBqv6! zL-*TNC$u*e;cF>!-)ITcsVw~YEabrFOge1K&k9iTUe5_#sQ0|yk#x)t!()BFw5l{M zM!qx^6f*%UaghG_bBY*=o|iH+31qMj;$i8Lp;Xx;Pt4G?iwH5)GggEnE>-?qL11^^ zy1ze5o`x&H35F7y*_EiVZ+UJ&V6zko2)MN|gaCdvi$i^bw2NWmg4vK;c8)fVmVewD z-PvNsR#B!R%rDxQW38apiFOq%`Ho4a|2g##7S!vUV_6UC^$D+#9a zLfRp3<`wlg!SU}s<~FVw*TZ_I7_tcnl}jq}5Rf8J|LK+q6*#WB`;qz9`rW<%xpRUD zAiJ{x5CD7tz(Xy?#CIA3V5DsX9|?Z{wC}tA^R~us*SUUJU<%&5yK&#SuWnm>n&dQu zZ}s?(ooMj;r+we~pO-Z5J9h&+|Ci%e@UHs+r;PXz1Kjz4aDLjqasKUC_!kbk@3j~R zyzT!yFud;=@Imj;TM9fYvX-ZJ1NdeC?f5?(CI51KH}Ubm9p5WPJivIJ#aeQ2k^n#L z-<|)rW99o8JHIjR9Wp<_Fb3+ofyeP{f&f45`xt+oq29+Z0+$DVPSF23QGI};tucOo zApk$^`#66d;M~Vi0+)Jz;ygUgc>rTwy4rtly#YV%`!Ii2Mt{TP`stxRtdrjN?kniV zQ2yliWc{*#d;hlr)i3XNY~sLrP=$KnJPwPn1nm45qkMnbe>?x1QGXV-emTEml-@(< z|A|%U+QIpEth&Pfe77P1TQ?R>Fz)X`zytoQb^XT4^(%Xb^KX@}2Lr=|#{Fy7Mxp7Y z!GhxZ)4mV*XT9k@U^4cDS^ER<|5TqIfH!7gJplh{-v|G*q;Mac6YqZke^6j}(18~s z7=LxZmInS#Mty(U_dD=s-g&', tls=tls_config) + + client = docker.DockerClient(base_url='unix://var/run/docker.sock') + print(json.dumps(client.info(), indent=2)) + # print(json.dumps(client.version(), indent=2)) + + print("Add File Image") + with gzip.open(self.dockerImageArchive, "rb") as imageZip: + imagedata = imageZip.read() + sortAppImg = client.images.load(imagedata) + print(sortAppImg) + + imagesList = client.images.list(all=True) + print ("Images found: ") + print(*imagesList, sep="\n") + + print("Create Container") + cmdValue = "{} {} {} {}".format(self.softwareExecuteCommandName, self.gcubeToken, self.fileItemId, self.tempFolderItemId) + print("CommandValue: " + cmdValue) + hostConf = client.api.create_host_config(auto_remove=True); + container = client.api.create_container( + image='sortapp-img:latest', + command=cmdValue, + host_config=hostConf) + + print("Start Container") + client.api.start(container=container.get('Id')) + client.api.wait(container=container.get('Id'), timeout=3600) + print("Container Execution End") + + def __str__(self): + return ('DEngine[storageHubUrl=' + str(self.storageHubUrl) + + ', softwareImage=' + str(self.softwareImage) + + ', softwareExecuteCommandName=' + str(self.softwareExecuteCommandName) + + ', fileItemId=' + str(self.fileItemId) + + ', tempFolderItemId=' + str(self.tempFolderItemId) + ']') diff --git a/engine/swarmengine.py b/engine/swarmengine.py new file mode 100644 index 0000000..75a8cb3 --- /dev/null +++ b/engine/swarmengine.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2020/06/12 +# +import docker +import json +import time +from docker.types.services import RestartPolicy + + +class SwarmEngine: + + def __init__(self, gcubeToken, storageHubUrl, softwareImage, softwareExecuteCommandName, fileItemId, tempFolderItemId): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + + # Software image for example: "microservices-VirtualBox:443/sortapp" + self.softwareImage = softwareImage + # Software Execute Command Name for example: "sortapp" + self.softwareExecuteCommandName = softwareExecuteCommandName + + # Input Data File + self.fileItemId = fileItemId + self.tempFolderItemId = tempFolderItemId + + + def monitorTask(self, srv): + print("Monitor Task") + + end = False + sId=None + + while not end: + for task in srv.tasks(): + print("Task: " + str(task)) + sId = task['ServiceID'] + if sId and sId == srv.id: + status = task['DesiredState'] + print("Task DesiredState: " + str(status)) + if status == 'shutdown' or status == 'complete': + print("Task End") + srv.remove() + print("Service Removed") + end = True + elif (status == 'failed' or status == 'rejected' + or status == 'orphaned' or status == 'remove'): + print("Error in execute Docker Image on Swarm Node: " + str(status)) + raise Exception("Error in execute Docker Image on Swarm Node: " + str(status)) + else: + time.sleep(2) + else: + continue + if sId==None: + print("Waiting Task load on Service") + time.sleep(2) + + def execute(self): + print("Execute SwarmRegistryEngine") + + print("Create Client") + + # DockerClient + # for example: http://microservices-VirtualBox:2375/ + # + # TLS support + # tls_config = docker.tls.TLSConfig( + # client_cert=('/path/to/client-cert.pem', '/path/to/client-key.pem') + # ) + # client = docker.DockerClient(base_url='', tls=tls_config) + + client = docker.DockerClient(base_url='http://docker-swarm1.int.d4science.net:2376/') + # client = docker.DockerClient(base_url='unix://var/run/docker.sock') + print(json.dumps(client.info(), indent=2)) + + image=client.images.pull(self.softwareImage, tag='latest') + + print("Pulled: "+str(image)) + + print("Services: " + str(client.services.list())) + + cmdValue = "{} {} {} {}".format(self.softwareExecuteCommandName, self.gcubeToken, self.fileItemId, self.tempFolderItemId) + print("CommandValue: " + cmdValue) + + print("Create Service") + + srv = client.services.create(self.softwareImage, command=cmdValue, + restart_policy=RestartPolicy(condition='none', delay=0, max_attempts=0, window=0)) + + print("Service: " + str(srv)) + + print("Service Id: " + str(srv.id)) + + print("Node: " + str(client.nodes.list())) + + self.monitorTask(srv) + + print("Service Execution End") + + def __str__(self): + return ('SwarmRegistryEngine[storageHubUrl=' + str(self.storageHubUrl) + + ', softwareImage=' + str(self.softwareImage) + + ', softwareExecuteCommandName=' + str(self.softwareExecuteCommandName) + + ', fileItemId=' + str(self.fileItemId) + + ', tempFolderItemId=' + str(self.tempFolderItemId) + ']') + diff --git a/issupport.py b/issupport.py new file mode 100644 index 0000000..536453d --- /dev/null +++ b/issupport.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import requests +from xml.etree import ElementTree + + +class ISSupport: + + def __init__(self): + # dev + # self.serviceUrl = "https://node10-d-d4s.d4science.org" + # prod + self.serviceUrl = "http://registry.d4science.org" + self.storageHubServiceClass = "DataAccess" + self.storageHubServiceName = "StorageHub" + + def discoverStorageHub(self, gcubeToken): + print("Discover StorageHub") + urlString = self.serviceUrl + "/icproxy/gcube/service/GCoreEndpoint/" + self.storageHubServiceClass + "/" + self.storageHubServiceName + "?gcube-token=" + gcubeToken + r = requests.get(urlString) + print(r.status_code) + print(r.text) + if r.status_code != 200: + print("Error discovering StorageHub: " + r.status_code) + raise Exception("Error retrieving StorageHub url info: " + r.status_code) + else: + root = ElementTree.fromstring(r.text) + print(root) + gcoreEndpoint = root.findall("Result/Resource/Profile/AccessPoint/RunningInstanceInterfaces/Endpoint") + print(gcoreEndpoint) + for child in gcoreEndpoint: + print(child.tag, child.attrib) + if child.attrib["EntryName"] == "org.gcube.data.access.storagehub.StorageHub": + print("Endpoint Found") + print(child.text) + return child.text + + print("Error discovering StorageHub url not found") + raise Exception("Error retrieving StorageHub url not found!") + + def __str__(self): + return 'ISSupport[serviceUrl=' + str(self.serviceUrl) + ']' diff --git a/storagehub/__init__.py b/storagehub/__init__.py new file mode 100644 index 0000000..79e232c --- /dev/null +++ b/storagehub/__init__.py @@ -0,0 +1 @@ +print("sortapp/command init") \ No newline at end of file diff --git a/storagehub/storagehubcommand.py b/storagehub/storagehubcommand.py new file mode 100644 index 0000000..85eb1d4 --- /dev/null +++ b/storagehub/storagehubcommand.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# + + +class StorageHubCommand: + + def execute(self): + print("StorageHubCommand") + + def __str__(self): + return 'StorageHubCommand' + diff --git a/storagehub/storagehubcommandcreatefolder.py b/storagehub/storagehubcommandcreatefolder.py new file mode 100644 index 0000000..186cad2 --- /dev/null +++ b/storagehub/storagehubcommandcreatefolder.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import requests + +from .storagehubcommand import StorageHubCommand + + +class StorageHubCommandCreateFolder(StorageHubCommand): + + def __init__(self, gcubeToken, storageHubUrl, folderItemId, folderName, folderDescription, folderHidden="false"): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + self.folderItemId = folderItemId + self.folderName = folderName + self.folderDescription = folderDescription + self.folderHidden = folderHidden + + def execute(self): + print("Execute StorageHubCommandCreateFolder") + print(self.storageHubUrl + "/items/" + self.folderItemId + "/create/FOLDER?"); + + folderdata = {'name': self.folderName, 'description': self.folderDescription, 'hidden': self.folderHidden} + print(str(folderdata)) + + urlString = self.storageHubUrl + "/items/" + self.folderItemId + "/create/FOLDER?gcube-token=" + self.gcubeToken + r = requests.post(urlString, data=folderdata) + print(r) + print(r.status_code) + if r.status_code != 200: + print("Error in execute StorageHubCommandCreateFolder: " + r.status_code) + raise Exception("Error in execute StorageHubCommandCreateFolder: " + r.status_code) + print('Created Folder ItemId: ' + str(r.text)) + return r.text + + def __str__(self): + return ('StorageHubCommandItemUpload[ storageHubUrl=' + str(self.storageHubUrl) + + ' folderItemId=' + str(self.folderItemId) + + ', folderName=' + str(self.folderName) + + ', folderDescription=' + str(self.folderDescription) + + ', folderHidden=' + str(self.folderHidden) + ']') diff --git a/storagehub/storagehubcommandcreatetempfolder.py b/storagehub/storagehubcommandcreatetempfolder.py new file mode 100644 index 0000000..8c8ca52 --- /dev/null +++ b/storagehub/storagehubcommandcreatetempfolder.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import uuid + +from .storagehubcommand import StorageHubCommand +from .storagehubcommandrootitemid import StorageHubCommandRootItemId +from .storagehubcommandcreatefolder import StorageHubCommandCreateFolder + + +class StorageHubCommandCreateTempFolder(StorageHubCommand): + + def __init__(self, gcubeToken, storageHubUrl): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + self.tempFolderName = "DataMinerResult" + self.tempFolderDescription = "DataMiner temporary result folder" + self.tempFolderHidden = "false" + + def execute(self): + print("Execute StorageHubCommandCreateTempFolder") + + print("Retrieve RootFolder") + cmdRootItemId = StorageHubCommandRootItemId(self.gcubeToken, self.storageHubUrl) + rootFolderItemId = cmdRootItemId.execute() + + print("Create Temp Folder Name") + tempFolderPostfix = uuid.uuid4().hex + self.tempFolderName = self.tempFolderName + tempFolderPostfix + print("Temp Folder Name: " + str(self.tempFolderName)) + + cmdCreateFolder = StorageHubCommandCreateFolder(self.gcubeToken, self.storageHubUrl, rootFolderItemId, + self.tempFolderName, self.tempFolderDescription, self.tempFolderHidden) + tempFolderItemId = cmdCreateFolder.execute() + + print("Temp Folder Item Id: " + str(tempFolderItemId)) + + return tempFolderItemId + + def __str__(self): + return 'StorageHubCommandCreateTempFolder[storageHubUrl=' + str(self.storageHubUrl) + ']' diff --git a/storagehub/storagehubcommanditemdelete.py b/storagehub/storagehubcommanditemdelete.py new file mode 100644 index 0000000..5104163 --- /dev/null +++ b/storagehub/storagehubcommanditemdelete.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import requests +from .storagehubcommand import StorageHubCommand + + +class StorageHubCommandItemDelete(StorageHubCommand): + + def __init__(self, gcubeToken, storageHubUrl, itemId, permanently="false"): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + self.itemId = itemId + self.permanently = permanently + + def execute(self): + print("Execute StorageHubCommandItemDelete") + print(self.storageHubUrl + "/items/" + self.itemId + "?force=" + self.permanently); + + urlString = self.storageHubUrl + "/items/" + self.itemId + "?force=" + self.permanently + "&gcube-token=" + self.gcubeToken + r = requests.delete(urlString) + print(r.status_code) + if r.status_code != 200: + print("Error in execute StorageHubCommandItemDelete: " + r.status_code) + raise Exception("Error in execute StorageHubCommandItemDelete: " + r.status_code) + + def __str__(self): + return ('StorageHubCommandItemDelete[storageHubUrl=' + str(self.storageHubUrl) + + ', itemId=' + self.itemId + + ', permanently=' + self.permanently + ']') diff --git a/storagehub/storagehubcommanditemdownload.py b/storagehub/storagehubcommanditemdownload.py new file mode 100644 index 0000000..d4ea1fe --- /dev/null +++ b/storagehub/storagehubcommanditemdownload.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import requests +from .storagehubcommand import StorageHubCommand + + +class StorageHubCommandItemDownload(StorageHubCommand): + + def __init__(self, gcubeToken, storageHubUrl, itemId, destinationFile): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + self.itemId = itemId + self.destinationFile = destinationFile + + def execute(self): + print("Execute StorageHubCommandItemDownload") + print(self.storageHubUrl + "/items/" + self.itemId + "/download?"); + + urlString = self.storageHubUrl + "/items/" + self.itemId + "/download?gcube-token=" + self.gcubeToken + r = requests.get(urlString) + print(r.status_code) + if r.status_code != 200: + print("Error in execute StorageHubCommandItemDownload: " + r.status_code) + raise Exception("Error in execute StorageHubCommandItemDownload: " + r.status_code) + with open(self.destinationFile, 'wb') as file: + file.write(r.content) + + def __str__(self): + return ('StorageHubCommandItemDownload[storageHubUrl=' + str(self.storageHubUrl) + + ', itemId=' + self.itemId + + ', destinationFile=' + self.destinationFile + ']') diff --git a/storagehub/storagehubcommanditemupload.py b/storagehub/storagehubcommanditemupload.py new file mode 100644 index 0000000..532e7dc --- /dev/null +++ b/storagehub/storagehubcommanditemupload.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import requests +from .storagehubcommand import StorageHubCommand + + +class StorageHubCommandItemUpload(StorageHubCommand): + + def __init__(self, gcubeToken, storageHubUrl, folderItemId, file, filename, fileDescription): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + self.folderItemId=folderItemId + self.file=file + self.filename=filename + self.fileDescription=fileDescription + + + def execute(self): + print("Execute StorageHubCommandItemUpload") + print(self.storageHubUrl + "/items/" + self.folderItemId + "/create/FILE?"); + + filedata = {'name': self.filename, 'description': self.fileDescription, "file": ("file", open(self.file, "rb"))} + + urlString = self.storageHubUrl + "/items/" + self.folderItemId + "/create/FILE?gcube-token=" + self.gcubeToken + r = requests.post(urlString, files=filedata) + print(r) + print(r.status_code) + if r.status_code != 200: + print("Error in execute StorageHubCommandItemUpload: " + r.status_code) + raise Exception("Error in execute StorageHubCommandItemUpload: " + r.status_code) + print(str(r.text)) + return r.text + + + def __str__(self): + return ('StorageHubCommandItemUpload[itemId='+self.folderItemId+ + ', storageHubUrl=' + str(self.storageHubUrl)+ + ', folderItemId=' + str(self.folderItemId) + + ', filename=' + str(self.filename)+ + ', fileDescription=' + str(self.fileDescription) + ']') diff --git a/storagehub/storagehubcommandrootitemid.py b/storagehub/storagehubcommandrootitemid.py new file mode 100644 index 0000000..6f587aa --- /dev/null +++ b/storagehub/storagehubcommandrootitemid.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @author: Giancarlo Panichi +# +# Created on 2018/06/15 +# +import requests +import json +from .storagehubcommand import StorageHubCommand + + +class StorageHubCommandRootItemId(StorageHubCommand): + + def __init__(self, gcubeToken, storageHubUrl): + self.gcubeToken = gcubeToken + self.storageHubUrl = storageHubUrl + + def execute(self): + print("Execute StorageHubCommandRootItemId") + print(self.storageHubUrl + "/?exclude=hl:accounting"); + urlString = self.storageHubUrl + "/?exclude=hl:accounting&gcube-token=" + self.gcubeToken + r = requests.get(urlString) + print(r.status_code) + if r.status_code != 200: + print("Error in execute StorageHubCommandRootItemId: " + r.status_code) + raise Exception("Error in execute StorageHubCommandRootItemId: " + r.status_code) + + rootItemJ = json.loads(r.text) + rootId = rootItemJ['item']['id'] + print('RootItemId:' + str(rootId)) + return rootId + + def __str__(self): + return 'StorageHubCommandRootItemId[storageHubUrl=' + str(self.storageHubUrl) + ']'