gCubeSystem
/
conductor-worker-python
17
0
Fork 0
Code Pull Requests Releases Activity

major refactoring

This commit is contained in:
dcore94 2020-10-14 18:06:28 +02:00
parent 984ecbab7b
commit 4526d0c999
17 changed files with 456 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -1,7 +1,7 @@
FROM python:3.7
COPY conductor/ /app/conductor
COPY pyexecplugins/ /app/pyexecplugins
COPY requirements.txt pyrest.py pyexec.py config.cfg entrypoint.sh /app/
COPY requirements.txt PyExec.py config.cfg entrypoint.sh /app/
RUN pip install -r /app/requirements.txt
WORKDIR /app
ENTRYPOINT ["/app/entrypoint.sh"]

100
PyExec.py Normal file
View File

@ -0,0 +1,100 @@
import configparser
import importlib
import logging
import sys
import os
import random
from pyexecplugins.pyexecplugins import *
from conductor.ConductorWorker import ConductorWorker
class PyExec():
errors = { "info" : logging.INFO, "error" : logging.ERROR, "debug" : logging.DEBUG}
def __init__(self, pluginlist):
self.init(pluginlist)
def computeDomain(self, plg):
domain = self.getDomain(plg)
commondomain = self.getDomain("common")
if domain is not None and commondomain is not None:
domain = domain + "," + commondomain
elif commondomain is not None:
domain = commondomain
return domain
def getDomain(self, plg):
if self.cfg.has_section(plg):
return self.cfg[plg].get("domain")
else:
return None
def init(self, pluginlist):
self.cfg = configparser.ConfigParser()
self.cfg.read("config.cfg")
self.workerid = self.cfg["common"].get("workerid", "pythonworker-" + str(random.randint(1000,10000)))
logging.basicConfig(level=self.errors[self.cfg["common"]["loglevel"]])
logging.info("Initializing PyExec worker %s with config.cfg", self.workerid)
self.threads = self.cfg["common"].getint("threads", 3)
self.pollrate = self.cfg["common"].getfloat("pollrate", .1)
self.server = os.environ.get('CONDUCTOR_SERVER', self.cfg["common"].get("server", "http://localhost:8080/api"))
for plg in pluginlist:
path = "pyexecplugins." + plg
try:
importlib.import_module(path)
except Exception as err:
logging.warning("Skipping plugin %s %s", path, err)
continue
PyExecPlugin.registerTaskDefinitions(self.server)
def start(self):
logging.info("Started PyExec with plugins: %s", PyExecPlugin.getPluginNames())
cc = ConductorWorker(self.server, self.threads, self.pollrate, self.workerid)
# start workers for all included plugins that are available also as standalone tasks ...
for plg in PyExecPlugin.getPlugins():
plugin = PyExecPlugin.get(plg)
if plugin.supportsStandalone(plugin):
domain = self.computeDomain(plugin.taskdef["name"])
cc.start(plugin.taskdef["name"], self.pyexec, False, domain)
# ... plus fallback pyexec for managing non standalone calls, Nop, Identity and Sequence
domain = self.computeDomain("pyexec")
cc.start('pyexec', self.pyexec, True, domain)
def pyexec(self, task):
try:
logging.info("Executing task {} of type {}[{}] from wkf {}[{}]".format(
task["workflowTask"]["taskReferenceName"],
task["taskDefName"],
task["taskId"],
task["workflowType"],
task["workflowInstanceId"]))
# Get operation from input or task type (in case of standalone tasks) fallback to Nop
operation = task["inputData"].get("operation") or task["taskDefName"] or "Nop"
logging.debug("Operation is %s", operation)
# Get plugin by checking either name or alias (in case of standalone tasks the task type is aliased to the plugin name)
p = PyExecPlugin.get(operation) or PyExecPlugin.getAlias(operation)
logging.debug("Plugin is %s", p)
if p != None:
pi = p(task["inputData"])
ret = pi.execute()
else: raise Exception("Operation {} not found.".format(operation))
return { "status" : "COMPLETED", "output" : ret, "logs" : ["one","two"]}
except Exception as exc:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.debug(exc_type, fname, exc_tb.tb_lineno)
logging.error(str(exc))
return { "status" : "FAILED", "output" : { "message" : str(exc) }, "logs" : ["one","two"]}
if __name__ == '__main__':
pyexec = PyExec(sys.argv[1:])
pyexec.start()

8
README.md
View File

@ -1,10 +1,14 @@
# Python workers for Conductor
Python worker for Conductor provides :
1. PYREST Comprehensive REST client
2. PYEXEC Execute code from available modules
1. PyExec: a plugin based set of workers for Netflix OSS Conductor written in Python3 for
## Install
### Example
```
python3 PyExec # Runs a bare PyExec supporting only Nop, Identity and Sequence plugins
python3 PyExec Http Ansible Packer #this runs PyExec with pyrest, pyansible and pypacker plugins enabled
python3 PyExec Shell Eval # runs a a PyExec with only pyshell and py eval enabled. pyshell executes shell commands, pyeval executes arbitrary Python3 code
```

3
conductor/conductor.py
View File

@ -62,7 +62,7 @@ class BaseClient(object):
theHeader = self.mergeTwoDicts(self.headers, headers)
if body is not None:
jsonBody = json.dumps(body, ensure_ascii=False)
jsonBody = json.dumps(body, ensure_ascii=True)
resp = requests.put(theUrl, params=queryParams, data=jsonBody, headers=theHeader)
else:
resp = requests.put(theUrl, params=queryParams, headers=theHeader)
@ -190,7 +190,6 @@ class TaskClient(BaseClient):
params['workerid'] = workerid
if domain is not None:
params['domain'] = domain
try:
return self.get(url, params)
except Exception as err:

12
config.cfg
View File

@ -1,13 +1,11 @@
[common]
server = http://nubis2.int.d4science.net:8080/api
#domain = comma separated list of task domains to be applied to all task types as fallback
[pyrest]
loglevel = info
server = http://conductorserver:8080/api
#server = http://localhost:8080/api
threads = 3
pollrate = .1
#domain = comma separated list of task domains to be applied to pyrest task type
#domain = comma separated list of task domains to be applied to all task types as fallback
domain = marco
[pyexec]
threads = 3
pollrate = .1
#domain = comma separated list of task domains to be applied to pyexec task type

5
entrypoint.sh
View File

@ -9,7 +9,6 @@ while [[ "$(curl -s -o /dev/null -L -w ''%{http_code}'' $BASE/health)" != 200 ]]
sleep 5
done
echo 'Starting workers...'
echo 'Starting default workers...'
python3 pyrest.py &
python3 pyexec.py
python3 PyExec.py Http Shell Eval

47
pyexec.py
View File

@ -1,47 +0,0 @@
from conductor.ConductorWorker import ConductorWorker
import configparser
import sys
import os
from pyexecplugins.pyexecplugins import *
from pyexecplugins.Ansible import Ansible
def pyexec(task):
try:
print("Executing task {} of type {}[{}] from wkf {}[{}]".format(task["workflowTask"]["taskReferenceName"], task["taskDefName"], task["taskId"], task["workflowType"], task["workflowInstanceId"]))
operation = task["inputData"].get("operation") or "Nop"
p = PyExecPlugin.get(operation)
if p != None:
pi = p(task["inputData"])
ret = pi.execute()
else: raise Exception("Operation {} not found.".format(operation))
return { "status" : "COMPLETED", "output" : ret, "logs" : ["one","two"]}
except Exception as exc:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
return { "status" : "FAILED", "output" : { "message" : "Internal error: {}".format(exc)}, "logs" : ["one","two"]}
def main():
print("Started PyExec with plugins:", PyExecPlugin.getPlugins())
cfg = configparser.ConfigParser()
cfg.read("config.cfg")
threads = cfg["pyexec"].getint("threads")
pollrate = cfg["pyexec"].getfloat("pollrate")
server = os.environ.get('CONDUCTOR_SERVER', cfg["common"]["server"])
domain = cfg["pyexec"].get("domain")
if domain is None:
domain = cfg["common"].get("domain")
elif cfg["common"].get("domain") is not None:
domain = domain + "," + cfg["common"].get("domain")
cc = ConductorWorker(server, threads, pollrate, "pyexec")
cc.start('pyexec', pyexec, True, domain)
if __name__ == '__main__':
main()

11
pyexecplugins/Ansible.py
View File

@ -1,6 +1,7 @@
from pyexecplugins.pyexecplugins import PyExecPlugin
import json
import shutil
import logging
import ansible.constants as C
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.module_utils.common.collections import ImmutableDict
@ -41,6 +42,14 @@ class ResultsCollectorJSONCallback(CallbackBase):
class Ansible(PyExecPlugin):
name = "Ansible"
taskdef = {
"name" : "pyansible",
"retryCount" : 0,
"description" : "Execute ansible playbook",
"inputKeys" : ["playbook", "hosts", "connection", "verbosity", "extra_vars", "gather_facts"],
"outputKeys" : ["ok", "failed", "unreachable"],
"ownerEmail" : "m.lettere@gmail.com"
}
def __init__(self, data=None):
super().__init__(data)
@ -93,7 +102,7 @@ class Ansible(PyExecPlugin):
# create data structure that represents our play
play_sources = loader.load(self.playbook)
for play_source in play_sources:
print(">>> Executing playsource:",play_source)
logging.debug("Executing playsource:",play_source)
play_source["gather_facts"] = self.gather_facts
# Create play object, playbook objects use .load instead of init or new methods,
# this will also automatically create the task objects from the info provided in play_source

21
pyexecplugins/Eval.py Normal file
View File

@ -0,0 +1,21 @@
from pyexecplugins.pyexecplugins import PyExecPlugin
class Plugin(PyExecPlugin):
name = "Eval"
taskdef = {
"name" : "pyeval",
"description" : "Execute arbitrary python code",
"inputKeys" : ["code"],
"outputKeys" : ["result"],
"ownerEmail" : "m.lettere@gmail.com"
}
def __init__(self, data=None):
super().__init__(data)
def execute(self):
code = self.data.get("code")
if code != None:
ret = eval(code, { "data" : self.data})
return ret

45
pyexecplugins/GatewaysToRestTask.py
View File

@ -1,45 +0,0 @@
from pyexecplugins.pyexecplugins import *
class GatewayToRestTask(PyExecPlugin):
name = "GatewayToRestTask"
def __init__(self, data=None):
super().__init__(data)
self.urls = data["gateway_list"]
def execute(self):
output = { "tasks" : [] , "urls": {}}
i = 0
for url in self.urls:
i += 1
trefname = "dt{}".format(i)
output["urls"][trefname] = {}
output["tasks"].append({
"name" : "pyrest",
"taskReferenceName" : trefname,
"type" : "SIMPLE",
"inputParameters" : {
"url" : "${workflow.input.keycloak_admin_url}",
"method" : "POST",
"expect" : [201, 409],
"headers" : {
"Authorization" : "Bearer ${authorize.output.body.access_token}",
"Content-Type" : "application/json"
},
"body" : {
"clientId": url.split("/")[2],
"baseUrl": url,
"enabled": True,
"clientAuthenticatorType": "client-secret",
"redirectUris": [url + "/*"],
"webOrigins": ["/*"],
"standardFlowEnabled": True,
"directAccessGrantsEnabled": True,
"publicClient": True,
"protocol": "openid-connect",
"attributes": {"login_theme": "d4science" },
"fullScopeAllowed": True
}
}
})
return output

72
pyexecplugins/Http.py Normal file
View File

@ -0,0 +1,72 @@
import requests
import json
import logging
from pyexecplugins.pyexecplugins import PyExecPlugin
class Plugin(PyExecPlugin):
name = "Http"
taskdef = {
"name" : "pyrest",
"description" : "Execute an HTTP request with pyrest worker",
"inputKeys" : ["url", "body", "contentType", "method", "accept", "headers", "connectionTimeout", "readTimeout"],
"outputKeys" : ["body", "status", "reason", "headers"],
"ownerEmail" : "m.lettere@gmail.com"
}
def __init__(self, data):
super().__init__(data)
self.method = data.get("method") or "get"
self.url = data.get("url")
self.headers = data.get("headers") or {}
self.contenttype = self.headers.get("Content-Type")
self.accept = self.headers.get("Accept")
self.body = data.get("body")
self.params = data.get("params")
self.expect = data.get("expect")
self.fail = data.get("fail")
def doRequest(self):
logging.debug(self.method, self.url, self.contenttype, self.accept)
if self.contenttype != None and self.contenttype.find("json") != -1:
self.response = requests.request(self.method, self.url, headers=self.headers, json = self.body)
else:
self.response = requests.request(self.method, self.url, headers=self.headers, data = self.body, params = self.params)
return self.response
def computeStatus(self):
if self.fail == False:
return "COMPLETED"
elif self.expect == None:
return "COMPLETED" if self.response.ok else "FAILED"
else:
if type(self.expect) == list:
return "COMPLETED" if (self.response.status_code in self.expect) else "FAILED"
else:
return "COMPLETED" if (self.response.status_code == self.expect) else "FAILED"
def buildOutput(self, status):
hdrs = {}
for k in self.response.headers.keys(): hdrs[k] = self.response.headers[k]
logging.info("Response: {} {}".format(self.response.status_code, self.response.reason))
if hdrs.get("Content-Type") != None and hdrs["Content-Type"].find("json") != -1 or self.accept != None and self.accept.find("json") != -1:
outbody = self.response.json() if len(self.response.content) != 0 else None
else:
outbody = self.response.text
logging.debug(outbody)
if status == "FAILED":
raise("HTTP call failed with status {} ({}) - {}".format(self.response.status_code, self.response.reson, str(outbody)))
return {
"body" : outbody,
"headers" : hdrs,
"status" : self.response.status_code,
"reason" : self.response.reason
}
def execute(self):
self.doRequest()
status = self.computeStatus()
return self.buildOutput(status)

42
pyexecplugins/Packer.py Normal file
View File

@ -0,0 +1,42 @@
from pyexecplugins.pyexecplugins import PyExecPlugin
import tempfile
import json
import subprocess
class Plugin(PyExecPlugin):
name = "Packer"
taskdef = {
"name" : "pypacker",
"retryCount" : 0,
"description" : "Executes packer.io command line for build and validate. It has been isolated in order to be able to start the worker only where OS dependencies are matched.",
"inputKeys" : ["command", "template"],
"outputKeys" : ["results"],
"ownerEmail" : "m.lettere@gmail.com"
}
def __init__(self, data=None):
super().__init__(data)
self.template = data["template"]
self.command = data.get("command", "build")
def execute(self):
fp = tempfile.NamedTemporaryFile(mode="w", delete=False)
fp.write(json.dumps(self.template))
fp.close()
completed = subprocess.run(["packer", self.command, fp.name], capture_output=True, text=True)
if(completed.returncode != 0):
raise Exception("packer.io failed: {} - out:{} - err:{}".format(
completed.returncode,
completed.stdout if completed.stdout != None else "",
completed.stderr if completed.stderr != None else "")
)
return {
'results' : [
{
'returncode' : completed.returncode,
'stdout' : completed.stdout,
'stderr': completed.stderr
}
]
}

35
pyexecplugins/Shell.py Normal file
View File

@ -0,0 +1,35 @@
import subprocess
import logging
from pyexecplugins.pyexecplugins import PyExecPlugin
class Plugin(PyExecPlugin):
name = "Shell"
taskdef = {
"name" : "pyshell",
"description" : "Execute an Shell commands on target machine. Commands are in the form of an array of objects like { 'line': 'ls -l', 'expect' : 0, 'withshell' : False}. For expect 0 is the default, for withshell False is the default thus they can be omitted.",
"inputKeys" : ["commands"],
"outputKeys" : ["results"],
"ownerEmail" : "m.lettere@gmail.com"
}
def __init__(self, data=None):
super().__init__(data)
def execute(self):
results = []
for cmdentry in self.data.get("commands"):
expect = cmdentry.get("expect", 0)
cmd = cmdentry.get("line")
#Use shell=True for very complex cmdlines that do have quotes and other amenities
withshell = cmdentry.get("withshell", self.data.get("withshell", False))
if cmd == None:
continue
else:
cmd = cmd.split() if not withshell else cmd
logging.debug("Going to execute (withshell=%b) %s", withshell, cmd)
completed = subprocess.run(cmd, shell=withshell, capture_output=True, text=True)
results.append({ 'returncode' : completed.returncode, 'stdout' : completed.stdout, 'stderr': completed.stderr})
if completed.returncode != expect:
raise Exception("Intermediate command failed with status code " + str(completed.returncode) )
return { "results": results }

279
pyexecplugins/pyexecplugins.py
View File

@ -1,28 +1,65 @@
import requests
import subprocess
import requests;
import json
class PyExecPlugins(type):
def __init__(cls, name, bases, attrs):
if not hasattr(cls, "plugins"):
print("Initializing plugins")
#print("Initializing plugins")
cls.plugins = {}
cls.alias = {}
else:
print("Appending a plugin ", cls.name, cls)
#print("Appending a plugin ", cls.name, cls)
cls.plugins[cls.name] = cls
#Alias plugin also with taskdefinition so it will be lookup-able with poth Operation name and taskdefinition
if cls.taskdef is not None:
cls.alias[cls.taskdef["name"]] = cls
def getPlugins(cls):
return cls.plugins
def getPluginNames(cls):
return [*cls.plugins.keys()]
def get(cls, name):
return cls.plugins.get(name)
def getAlias(cls, name):
return cls.alias.get(name)
def registerTaskDefinitions(cls, server):
url = server + "/metadata/taskdefs"
#pyexec generic taskdefinition
taskdefs = [
{
"name" : "pyexec",
"description" : "Execute PyExec operations",
"inputKeys" : ["operation"],
"outputKeys" : ["ret"],
"ownerEmail" : "m.lettere@gmail.com"
}
]
for plg in cls.getPluginNames():
if cls.plugins[plg].taskdef is not None:
taskdef = cls.plugins[plg].taskdef
taskdefs.append(taskdef)
#Post all new (or not) task definitions to orchestrator
headers = {'Content-Type': 'application/json'}
try:
response = requests.request("POST", url, headers=headers, data=json.dumps(taskdefs))
except Exception as e:
print("Unable to register task defs", e)
class PyExecPlugin(object, metaclass=PyExecPlugins):
def __init__(self, data=None):
self.data = data
def supportsStandalone(self):
return (self.taskdef is not None)
class Nop(PyExecPlugin):
name = "Nop"
taskdef = None
def __init__(self, data=None):
super().__init__(data)
@ -32,6 +69,7 @@ class Nop(PyExecPlugin):
class Identity(PyExecPlugin):
name = "Identity"
taskdef = None
def __init__(self, data=None):
super().__init__(data)
@ -42,6 +80,7 @@ class Identity(PyExecPlugin):
class Sequence(PyExecPlugin):
name = "Sequence"
taskdef = None
def __init__(self, data=None):
super().__init__(data)
@ -66,121 +105,121 @@ class Sequence(PyExecPlugin):
self.handleSequence(self.data.get("tasks",[]),self.result)
return {"result": self.result}
class Http(PyExecPlugin):
name = "Http"
def __init__(self, data):
super().__init__(data)
self.method = data.get("method") or "get"
self.url = data.get("url")
self.headers = data.get("headers") or {}
self.contenttype = self.headers.get("Content-Type")
self.accept = self.headers.get("Accept")
self.body = data.get("body")
self.params = data.get("params")
self.expect = data.get("expect")
self.fail = data.get("fail")
def doRequest(self):
#print(self.method, self.url, self.contenttype, self.accept)
if self.contenttype != None and self.contenttype.find("json") != -1:
self.response = requests.request(self.method, self.url, headers=self.headers, json = self.body)
else:
self.response = requests.request(self.method, self.url, headers=self.headers, data = self.body, params = self.params)
return self.response
def computeStatus(self):
if self.fail == False:
return "COMPLETED"
elif self.expect == None:
return "COMPLETED" if self.response.ok else "FAILED"
else:
if type(self.expect) == list:
return "COMPLETED" if (self.response.status_code in self.expect) else "FAILED"
else:
return "COMPLETED" if (self.response.status_code == self.expect) else "FAILED"
def buildOutput(self, status):
hdrs = {}
for k in self.response.headers.keys(): hdrs[k] = self.response.headers[k]
print("Response: {} {}".format(self.response.status_code, self.response.reason))
if hdrs.get("Content-Type") != None and hdrs["Content-Type"].find("json") != -1 or self.accept != None and self.accept.find("json") != -1:
outbody = self.response.json() if len(self.response.content) != 0 else None
else:
outbody = self.response.text
#print(outbody)
return {
'status': status,
'output': {
"body" : outbody,
"headers" : hdrs,
"status" : self.response.status_code,
"reason" : self.response.reason
},
'logs': ['one', 'two']
}
def execute(self):
self.doRequest()
status = self.computeStatus()
return self.buildOutput(status)
#class Http(PyExecPlugin):
# name = "Http"
#
# def __init__(self, data):
# super().__init__(data)
# self.method = data.get("method") or "get"
# self.url = data.get("url")
# self.headers = data.get("headers") or {}
# self.contenttype = self.headers.get("Content-Type")
# self.accept = self.headers.get("Accept")
# self.body = data.get("body")
# self.params = data.get("params")
# self.expect = data.get("expect")
# self.fail = data.get("fail")
#
# def doRequest(self):
# #print(self.method, self.url, self.contenttype, self.accept)
# if self.contenttype != None and self.contenttype.find("json") != -1:
# self.response = requests.request(self.method, self.url, headers=self.headers, json = self.body)
# else:
# self.response = requests.request(self.method, self.url, headers=self.headers, data = self.body, params = self.params)
# return self.response
#
# def computeStatus(self):
# if self.fail == False:
# return "COMPLETED"
# elif self.expect == None:
# return "COMPLETED" if self.response.ok else "FAILED"
# else:
# if type(self.expect) == list:
# return "COMPLETED" if (self.response.status_code in self.expect) else "FAILED"
# else:
# return "COMPLETED" if (self.response.status_code == self.expect) else "FAILED"
#
# def buildOutput(self, status):
# hdrs = {}
# for k in self.response.headers.keys(): hdrs[k] = self.response.headers[k]
#
# print("Response: {} {}".format(self.response.status_code, self.response.reason))
#
# if hdrs.get("Content-Type") != None and hdrs["Content-Type"].find("json") != -1 or self.accept != None and #self.accept.find("json") != -1:
# outbody = self.response.json() if len(self.response.content) != 0 else None
# else:
# outbody = self.response.text
# #print(outbody)
# return {
# 'status': status,
# 'output': {
# "body" : outbody,
# "headers" : hdrs,
# "status" : self.response.status_code,
# "reason" : self.response.reason
# },
# 'logs': ['one', 'two']
# }
#
# def execute(self):
# self.doRequest()
# status = self.computeStatus()
# return self.buildOutput(status)
class Eval(PyExecPlugin):
name = "Eval"
def __init__(self, data=None):
super().__init__(data)
def execute(self):
code = self.data.get("code")
if code != None:
ret = eval(code, { "data" : self.data})
return { "result" : ret }
#class Eval(PyExecPlugin):
# name = "Eval"
#
# def __init__(self, data=None):
# super().__init__(data)
#
# def execute(self):
# code = self.data.get("code")
# if code != None:
# ret = eval(code, { "data" : self.data})
# return { "result" : ret }
class Write(PyExecPlugin):
name = "Write"
def __init__(self, data=None):
super().__init__(data)
def execute(self):
print("File opening at", self.data["path"])
fo = open(self.data["path"], "w")
print("File opened", fo)
typ = self.data.get("type", "text")
if typ == "json":
fo.write(json.dumps(self.data.get("content","{}")))
else:
fo.write(self.data.get("content", ""))
fo.close()
return {}
#class Write(PyExecPlugin):
# name = "Write"
#
# def __init__(self, data=None):
# super().__init__(data)
#
# def execute(self):
# print("File opening at", self.data["path"])
# fo = open(self.data["path"], "w")
# print("File opened", fo)
# typ = self.data.get("type", "text")
# if typ == "json":
# fo.write(json.dumps(self.data.get("content","{}")))
# else:
# fo.write(self.data.get("content", ""))
# fo.close()
# return {}
class Shell(PyExecPlugin):
name = "Shell"
def __init__(self, data=None):
super().__init__(data)
def execute(self):
output = {
'results' : [],
'status' : "COMPLETED"
}
for cmdentry in self.data.get("commands"):
expect = cmdentry.get("expect", 0)
cmd = cmdentry.get("line")
#Use shell=True for very complex cmdlines that do have quotes and ther amenities
withshell = cmdentry.get("withshell", self.data.get("withshell", False))
if cmd == None:
continue
else:
cmd = cmd.split() if not withshell else cmd
print("Going to execute", withshell, cmd)
completed = subprocess.run(cmd, shell=withshell, capture_output=True, text=True)
output["results"].append({ 'returncode' : completed.returncode, 'stdout' : completed.stdout, 'stderr': completed.stderr})
if completed.returncode != expect:
output["status"] = "FAILED"
break
return output
#class Shell(PyExecPlugin):
# name = "Shell"
#
# def __init__(self, data=None):
# super().__init__(data)
#
# def execute(self):
# output = {
# 'results' : [],
# 'status' : "COMPLETED"
# }
# for cmdentry in self.data.get("commands"):
# expect = cmdentry.get("expect", 0)
# cmd = cmdentry.get("line")
# #Use shell=True for very complex cmdlines that do have quotes and ther amenities
# withshell = cmdentry.get("withshell", self.data.get("withshell", False))
# if cmd == None:
# continue
# else:
# cmd = cmd.split() if not withshell else cmd
# print("Going to execute", withshell, cmd)
# completed = subprocess.run(cmd, shell=withshell, capture_output=True, text=True)
# output["results"].append({ 'returncode' : completed.returncode, 'stdout' : completed.stdout, 'stderr': completed.stderr})
# if completed.returncode != expect:
# output["status"] = "FAILED"
# break
# return output

43
pyrest.py
View File

@ -1,43 +0,0 @@
from conductor.ConductorWorker import ConductorWorker
from pyexecplugins.pyexecplugins import Http
import configparser
import sys
import os
def log(task, message):
print("[{}:{}][{}:{}] - {}".format(task["workflowType"], task["workflowInstanceId"], task["taskDefName"], task["workflowTask"]["taskReferenceName"], message))
def pyrest(task):
log(task, "Starting execution ...")
log(task, "Data is {} {} {}".format(task["inputData"].get("method"), task["inputData"]["url"], task["inputData"].get("body")))
http = Http(task["inputData"])
try:
ret = http.execute()
return ret
except Exception as exc:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
return {
"status" : "FAILED",
"output" : { "message" : "Internal error: {}".format(exc)}, "logs" : ["one","two"]
}
def main():
cfg = configparser.ConfigParser()
cfg.read("config.cfg")
threads = cfg["pyrest"].getint("threads")
pollrate = cfg["pyrest"].getfloat("pollrate")
server = os.environ.get('CONDUCTOR_SERVER', cfg["common"]["server"])
domain = cfg["pyrest"].get("domain")
if domain is None:
domain = cfg["common"].get("domain")
elif cfg["common"].get("domain") is not None:
domain = domain + "," + cfg["common"].get("domain")
cc = ConductorWorker(server, threads, pollrate, "pyrest")
cc.start('pyrest', pyrest, True, domain)
if __name__ == '__main__':
main()

1
requirements.txt
View File

@ -1,3 +1,2 @@
ansible >= 2.8.3
requests >= 2.19.1

4
setup.py
View File

@ -18,7 +18,7 @@ from setuptools import setup
setup(
name = 'nwconductorworkers',
packages = ['nwconductorworkers'], # this must be the same as the name above
version = '1.0.0',
version = '2.0.0',
description = 'Python based Conductor Workers',
author = 'Marco Lettere',
author_email = 'marco.lettere@nubisware.com',
@ -42,4 +42,4 @@ setup(
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: System :: Networking'
],
)
)