From ae534f7b6b9d23abc46d2d3f8c8a303d254fcf78 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 13 Feb 2019 00:28:26 -0500 Subject: [PATCH 1/3] initial k8 support --- python/phylanx/ast/clusters.py | 55 ++++++++++++++++++++++++++++++++ python/phylanx/ast/transducer.py | 17 +++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 python/phylanx/ast/clusters.py diff --git a/python/phylanx/ast/clusters.py b/python/phylanx/ast/clusters.py new file mode 100644 index 000000000..d49fc23da --- /dev/null +++ b/python/phylanx/ast/clusters.py @@ -0,0 +1,55 @@ +# Copyright (c) 2018 Christopher Taylor +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +''' +see this documentation: + +https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/#accessing-the-cluster-api +https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ +https://github.com/kubernetes-client/python/blob/master/examples/exec.py +''' + +import requests +from base64 import b64encode + +class cluster(object): + def __init_(self, args): + self.args = args + + def send(self): + raise NotImplementedError("cluster::send not implemented.") + +class k8cluster(cluster): + def __init__(self, args): + cluster.__init__(self, args) + + @overrides(cluster) + def sends(self, physl_ir_str): + if 'jobconfig' not in self.args: + raise Error("k8cluster jobconfig not defined.") + if 'url' not in self.args: + raise Error("k8cluster url not defined.") + + env_ir = [ {'name' : 'PHYSL_IR', 'value' : b64encode(physl_ir_str) } ] + + if 'env' not in self.args['jobconfig']['spec']['containers']: + self.args['jobconfig']['spec']['containers']['env'] = list() + + self.args['jobconfig']['spec']['containers']['env'].append(env_ir) + + resp = requests.post(url=self.args['url'], args=self.args['jobconfig']) + return resp.json() + +def create_cluster(args): + if 'type' not in args: + raise Error("create_cluster type not defined.") + + cluster_type = args['type'] + + if 'jobconfig' not in args: + raise Error("create_cluster jobconfig not defined.") + + if cluster_type == 'k8' or cluster_type == 'kubernetes': + return k8cluster(args['jobconfig']) diff --git a/python/phylanx/ast/transducer.py b/python/phylanx/ast/transducer.py index 2d8dd5a61..fa07d4916 100644 --- a/python/phylanx/ast/transducer.py +++ b/python/phylanx/ast/transducer.py @@ -14,7 +14,7 @@ from phylanx import execution_tree from phylanx.ast import generate_ast as generate_phylanx_ast from phylanx.exceptions import InvalidDecoratorArgumentError - +from phylanx.clusters import create_cluster def Phylanx(__phylanx_arg=None, **kwargs): class __PhylanxDecorator(object): @@ -22,7 +22,7 @@ def __init__(self, f): """ :function:f the decorated funtion. """ - valid_kwargs = ['debug', 'target', 'compiler_state', 'performance'] + valid_kwargs = ['debug', 'target', 'compiler_state', 'performance', 'cluster'] self.backends_map = {'PhySL': PhySL, 'OpenSCoP': OpenSCoP} self.backend = self.get_backend(kwargs.get('target')) @@ -36,6 +36,14 @@ def __init__(self, f): else: kwargs['fglobals'] = f.__globals__ + # determine if code is sent to a remote cluster + # + self.remote = None + if 'cluster' in kwargs.keys(): + self.remote = create_cluster(kwargs['cluster']) + else: + raise NotImplementedError("Unknown Phylanx argument '%s'" % (kwargs['cluster'],)) + python_src = self.get_python_src(f) python_ast = self.get_python_ast(python_src, f) @@ -79,10 +87,11 @@ def __call__(self, *args): raise NotImplementedError( "OpenSCoP kernels are not yet callable.") - result = self.backend.call(args) + if self.remote is not None or self.remote != None: + return self.remote.send(self.__src__) + result = self.backend.call(args) self.__perfdata__ = self.backend.__perfdata__ - return result def generate_ast(self): From 38767c30c56c897ce42023292043deef6b0cb46c Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 13 Feb 2019 00:31:00 -0500 Subject: [PATCH 2/3] removed useless conditional --- python/phylanx/ast/transducer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/phylanx/ast/transducer.py b/python/phylanx/ast/transducer.py index fa07d4916..e344d7e1d 100644 --- a/python/phylanx/ast/transducer.py +++ b/python/phylanx/ast/transducer.py @@ -41,8 +41,6 @@ def __init__(self, f): self.remote = None if 'cluster' in kwargs.keys(): self.remote = create_cluster(kwargs['cluster']) - else: - raise NotImplementedError("Unknown Phylanx argument '%s'" % (kwargs['cluster'],)) python_src = self.get_python_src(f) python_ast = self.get_python_ast(python_src, f) From 5f79f532f6d80ea9638eb67155ed08930e3e8b41 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 13 Feb 2019 00:31:42 -0500 Subject: [PATCH 3/3] updated date --- python/phylanx/ast/clusters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/phylanx/ast/clusters.py b/python/phylanx/ast/clusters.py index d49fc23da..a9bea64db 100644 --- a/python/phylanx/ast/clusters.py +++ b/python/phylanx/ast/clusters.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Christopher Taylor +# Copyright (c) 2019 Christopher Taylor # # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)