From 86fb7740a101fd8c45c155e26d34c726685bb485 Mon Sep 17 00:00:00 2001 From: jem Date: Tue, 3 Mar 2020 16:34:12 +0100 Subject: [PATCH] new layout with base build & mixins --- build.py | 2 +- src/main/python/ddadevops/__init__.py | 12 +- src/main/python/ddadevops/aws_mixin.py | 37 ++++++ src/main/python/ddadevops/dda_pallet.py | 46 ------- src/main/python/ddadevops/dda_pallet_mixin.py | 68 ++++++++++ src/main/python/ddadevops/devops_build.py | 28 ++-- .../ddadevops/devops_terraform_build.py | 121 ++++++++---------- src/main/python/ddadevops/hetzner_mixin.py | 19 ++- 8 files changed, 194 insertions(+), 139 deletions(-) create mode 100644 src/main/python/ddadevops/aws_mixin.py delete mode 100644 src/main/python/ddadevops/dda_pallet.py create mode 100644 src/main/python/ddadevops/dda_pallet_mixin.py diff --git a/build.py b/build.py index f3f2c67..cc1f02e 100644 --- a/build.py +++ b/build.py @@ -27,7 +27,7 @@ use_plugin("python.distutils") default_task = "publish" name = "ddadevops" -version = "0.3.7.dev4" +version = "0.4.0.dev1" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] diff --git a/src/main/python/ddadevops/__init__.py b/src/main/python/ddadevops/__init__.py index 9a2ba38..dc20688 100644 --- a/src/main/python/ddadevops/__init__.py +++ b/src/main/python/ddadevops/__init__.py @@ -4,13 +4,11 @@ terraform, dda-pallet, aws & hetzner-cloud. """ -from .build import init_project, stage, name, module, \ - project_root_path, build_commons_path, build_target_path, \ - initialize_target, tf_import_name, tf_import_resource -from .meissa_build import meissa_init_project, hetzner_api_key from .credential import gopass_credential_from_env_path, gopass_credential_from_path -from .dda_pallet import dda_write_target, dda_write_domain, dda_install, dda_configure -from .terraform import tf_copy_common, tf_plan, tf_import, tf_apply, tf_output, tf_destroy, \ - tf_read_output_json +from .devops_build import DevopsBuild, create_devops_build_config +from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config +from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config +from .aws_mixin import AwsMixin, add_aws_mixin_config +from .dda_pallet_mixin import DdaPalletMixin, add_dda_pallet_mixin_config __version__ = "${version}" \ No newline at end of file diff --git a/src/main/python/ddadevops/aws_mixin.py b/src/main/python/ddadevops/aws_mixin.py new file mode 100644 index 0000000..870267c --- /dev/null +++ b/src/main/python/ddadevops/aws_mixin.py @@ -0,0 +1,37 @@ +from python_terraform import * +from .credential import gopass_credential_from_env_path +from .devops_terraform_build import DevopsTerraformBuild + + +def add_aws_mixin_config(config, account_name): + return config.update({'AwsMixin': + {'account_name': account_name}}) + + +class AwsMixin(DevopsTerraformBuild): + + def __init__(self, project, config): + super().__init__(self, project, config) + aws_mixin_config = config['AwsMixin'] + self.account_name = aws_mixin_config['account_name'] + + def backend_config(self): + return "backend." + self.account_name + "." + self.stage + ".properties" + + def project_vars(self): + ret = super().project_vars() + return ret.update({'account_name': self.account_name}) + + def init_client(self): + tf = Terraform(working_dir=self.build_path()) + tf.init(backend_config=self.backend_config) + try: + tf.workspace('select', slef.stage) + except: + tf.workspace('new', self.stage) + return tf + + def plan(self): + tf = self.init_client() + tf.plan(capture_output=False, var=self.project_vars, + var_file=self.backend_config) diff --git a/src/main/python/ddadevops/dda_pallet.py b/src/main/python/ddadevops/dda_pallet.py deleted file mode 100644 index 14450b5..0000000 --- a/src/main/python/ddadevops/dda_pallet.py +++ /dev/null @@ -1,46 +0,0 @@ -from .python_util import * -from .meissa_build import build_target_path, project_root_path -from string import Template - - -TARGET = 'target.edn' -TEMPLATE_TARGET_CONTENT = Template(""" -{:existing [{:node-name "k8s" - :node-ip "$ipv4"}] - :provisioning-user {:login "root"}} -""") - -def dda_write_target(project, ipv4): - with open(build_target_path(project) + TARGET, "w") as output_file: - output_file.write(TEMPLATE_TARGET_CONTENT.substitute({'ipv4' : ipv4})) - -def dda_write_domain(project, domain_file_name, substitues): - with open(build_target_path(project) + domain_file_name, "r") as input_file: - domain_input = input_file.read() - domain_template = Template(domain_input) - with open(build_target_path(project) + 'out_' + domain_file_name, "w") as output_file: - output_file.write(domain_template.substitute(substitues)) - -def dda_install(project, tenant, application, domain_file_name): - return dda_uberjar(project, tenant, application, domain_file_name) - -def dda_configure(project, tenant, application, domain_file_name): - return dda_uberjar(project, tenant, application, domain_file_name, True) - -def dda_uberjar(project, tenant, application, domain_file_name, configure_switch=None): - if configure_switch: - cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \ - '--targets', build_target_path(project) + TARGET, \ - '--tenant', tenant, '--application', application, \ - '--configure', \ - build_target_path(project) + 'out_' + domain_file_name] - else: - cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \ - '--targets', build_target_path(project) + TARGET, \ - '--tenant', tenant, '--application', application, \ - build_target_path(project) + 'out_' + domain_file_name] - prn_cmd=list(cmd) - print(" ".join(prn_cmd)) - output = execute(cmd) - print(output) - return output diff --git a/src/main/python/ddadevops/dda_pallet_mixin.py b/src/main/python/ddadevops/dda_pallet_mixin.py new file mode 100644 index 0000000..b82105d --- /dev/null +++ b/src/main/python/ddadevops/dda_pallet_mixin.py @@ -0,0 +1,68 @@ +from string import Template +from .python_util import * +from .devops_build import DevopsBuild + + +def add_dda_pallet_mixin_config(config, tenant, application, domain_file_name): + return config.update({'DdaPalletMixin': + {'tenant': tenant, + 'application': application, + 'domain_file_name': domain_file_name, + 'target_edn_name': 'target.edn', + 'jar_file': 'target/meissa-tenant-server.jar', + 'target_template': + """ +{:existing [{:node-name "$node_name" + :node-ip "$ipv4"}] + :provisioning-user {:login "root"}} +""", }}) + + +class DdaPalletMixin(DevopsBuild): + + def __init__(self, project, config): + super().__init__(self, project, config) + dda_pallet_mixin_config = config['DdaPalletMixin'] + self.tenant = dda_pallet_mixin_config['tenant'] + self.application = dda_pallet_mixin_config['application'] + self.domain_file_name = dda_pallet_mixin_config['domain_file_name'] + self.target_edn_name = dda_pallet_mixin_config['target_edn_name'] + self.jar_file = dda_pallet_mixin_config['jar_file'] + self.target_template = Template( + dda_pallet_mixin_config['target_template']) + + def dda_write_target(self, node_name, ipv4): + with open(self.build_path() + self.target_edn_name, "w") as output_file: + output_file.write( + self.target_template.substitute({'ipv4': ipv4, 'node_name': node_name})) + + def dda_write_domain(self, domain_file_name, substitues): + with open(self.build_path() + domain_file_name, "r") as input_file: + domain_input = input_file.read() + domain_template = Template(domain_input) + with open(self.build_path() + 'out_' + domain_file_name, "w") as output_file: + output_file.write(domain_template.substitute(substitues)) + + def dda_uberjar(self, configure_switch=None): + if configure_switch: + cmd = ['java', '-jar', self.project_root_path() + self.jar_file, + '--targets', self.build_path() + self.target_edn_name, + '--tenant', self.tenant, '--application', self.application, + '--configure', + self.build_path() + 'out_' + self.domain_file_name] + else: + cmd = ['java', '-jar', self.project_root_path() + self.jar_file, + '--targets', self.build_path() + self.target_edn_name, + '--tenant', self.tenant, '--application', self.application, + self.build_path() + 'out_' + self.domain_file_name] + prn_cmd = list(cmd) + print(" ".join(prn_cmd)) + output = execute(cmd) + print(output) + return output + + def dda_install(self): + return self.dda_uberjar() + + def dda_configure(self): + return self.dda_uberjar(True) diff --git a/src/main/python/ddadevops/devops_build.py b/src/main/python/ddadevops/devops_build.py index 8c84ca3..32f86bf 100644 --- a/src/main/python/ddadevops/devops_build.py +++ b/src/main/python/ddadevops/devops_build.py @@ -1,17 +1,25 @@ -from .credential import gopass_credential_from_env_path from subprocess import run + +def create_devops_build_config(stage, project_root_path, build_commons_path, module): + return {'stage': stage, + 'project_root_path': project_root_path, + 'build_commons_path': build_commons_path, + 'module': module, + 'build_dir_name': 'target'} + + class DevopsBuild: - def __init__(self, project, project_root_path, build_commons_path, module, stage): - self.stage = stage - self.project_root_path = project_root_path - self.build_commons_path = build_commons_path - self.module = module + def __init__(self, project, config): + self.stage = config['stage'] + self.project_root_path = config['project_root_path'] + self.build_commons_path = config['build_commons_path'] + self.module = config['module'] + self.build_dir_name = config['build_dir_name'] self.project = project - self.build_dir_name = 'target' project.set_property("devops_build", self) - + def name(self): return self.project.get_property('name') @@ -20,12 +28,12 @@ class DevopsBuild: def initialize_build_dir(self): run('rm -rf ' + self.build_path(), shell=True) - run('mkdir -p ' + self.build_path(), shell=True) + run('mkdir -p ' + self.build_path(), shell=True) def tf_import_name(project): return project.get_property('tf_import_name') + def tf_import_resource(project): return project.get_property('tf_import_resource') - diff --git a/src/main/python/ddadevops/devops_terraform_build.py b/src/main/python/ddadevops/devops_terraform_build.py index 6b05160..c9fe2ae 100644 --- a/src/main/python/ddadevops/devops_terraform_build.py +++ b/src/main/python/ddadevops/devops_terraform_build.py @@ -1,26 +1,37 @@ from os import path from json import load from subprocess import run -from .devops_build import DevopsBuild -from .python_util import execute from python_terraform import * +from .devops_build import DevopsBuild, create_devops_build_config + + +def create_devops_terraform_build_config(stage, project_root_path, build_commons_path, module, + account_name, additional_vars, tf_import_name, tf_import_resource): + ret = create_devops_build_config( + stage, project_root_path, build_commons_path, module) + return ret.update({'account_name': account_name, + 'additional_vars': additional_vars, + 'tf_import_name': tf_import_name, + 'tf_import_resource': tf_import_resource, + 'terraform_build_commons_dir_name': 'terraform', + 'output_json_name': 'output.json'}) + class DevopsTerraformBuild(DevopsBuild): - def __init__(self, project, project_root_path, build_commons_path, module, stage, account_name, additional_vars): - super().__init__(self, project, project_root_path, build_commons_path, module, stage) - self.account_name = account_name - self.additional_vars = additional_vars - self.terraform_build_commons_dir_name= 'terraform' - - def backend_config(self): - return "backend." + self.account_name + "." + self.stage + ".properties" + def __init__(self, project, config): + super().__init__(self, project, config) + self.additional_vars = config['additional_vars'] + self.tf_import_name = config['tf_import_name'] + self.tf_import_resource = config['tf_import_resource'] + self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name'] + self.output_json_name = config['output_json_name'] def terraform_build_commons_path(self): return self.build_commons_path() + '/' + self.terraform_build_commons_dir_name def project_vars(self): - ret = {'stage' : self.stage} + ret = {'stage': self.stage} if self.module: ret['module'] = self.module if self.additional_vars: @@ -29,77 +40,47 @@ class DevopsTerraformBuild(DevopsBuild): def initialize_build_dir(self): super().initialize_build_dir() - run('cp -f ' + self.terraform_build_commons_path + '* ' + self.build_path, shell=True) - run('cp *.tf ' + self.build_path, shell=True) + run('cp -f ' + self.terraform_build_commons_path + + '* ' + self.build_path, shell=True) + run('cp *.tf ' + self.build_path, shell=True) run('cp *.properties ' + self.build_path, shell=True) - run('cp *.tfars ' + self.build_path, shell=True) - run('cp *.edn ' + self.build_path, shell=True) + run('cp *.tfars ' + self.build_path, shell=True) + run('cp *.edn ' + self.build_path, shell=True) def init_client(self): tf = Terraform(working_dir=self.build_path()) - tf.init(backend_config=self.backend_config) + tf.init() try: tf.workspace('select', slef.stage) except: tf.workspace('new', self.stage) return tf + def write_output(self, tf): + result = tf.output(json=IsFlagged) + with open(self.build_path() + self.output_json_name, "w") as output_file: + output_file.write(json.dumps(result)) + + def read_output_json(self): + with open(self.build_path() + self.output_json_name, 'r') as f: + return load(f) + def plan(self): tf = self.init_client() - tf.plan(capture_output=False, var=self.project_vars, var_file=self.backend_config) + tf.plan(capture_output=False, var=self.project_vars) + def apply(self, p_auto_approve=False): + tf = self.init_client() + tf.apply(capture_output=False, auto_approve=p_auto_approve, + var=self.project_vars()) + self.write_output(tf) + def destroy(self, p_auto_approve=False): + tf = self.init_client() + tf.destroy(capture_output=False, auto_approve=p_auto_approve, + var=self.project_vars()) -OUTPUT_JSON = "output.json" - -def tf_copy_common(project): - run(['cp', '-f', build_commons_path(project) + 'terraform/aws_provider.tf', \ - build_target_path(project) + 'aws_provider.tf']) - run(['cp', '-f', build_commons_path(project) + 'terraform/variables.tf', \ - build_target_path(project) + 'variables.tf']) - -def tf_plan(project): - init(project) - tf = Terraform(working_dir=build_target_path(project)) - tf.plan(capture_output=False, var=project_vars(project)) - -def tf_import(project): - init(project) - tf = Terraform(working_dir=build_target_path(project)) - tf.import_cmd(tf_import_name(project), tf_import_resource(project), \ - capture_output=False, var=project_vars(project)) - -def tf_apply(project, p_auto_approve=False): - init(project) - tf = Terraform(working_dir=build_target_path(project)) - tf.apply(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project)) - tf_output(project) - -def tf_output(project): - init(project) - tf = Terraform(working_dir=build_target_path(project)) - result = tf.output(json=IsFlagged) - with open(build_target_path(project) + OUTPUT_JSON, "w") as output_file: - output_file.write(json.dumps(result)) - -def tf_destroy(project, p_auto_approve=False): - init(project) - tf = Terraform(working_dir=build_target_path(project)) - tf.destroy(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project)) - -def tf_read_output_json(project): - with open(build_target_path(project) + OUTPUT_JSON, 'r') as f: - return load(f) - -def project_vars(project): - my_hetzner_api_key = hetzner_api_key(project) - my_module = project.name - ret = {'stage' : stage(project)} - # TODO: move to meissa specific part - if my_hetzner_api_key: - ret['hetzner_api_key'] = my_hetzner_api_key - if my_module: - ret['module'] = my_module - return ret - - + def tf_import(self): + tf = self.init_client() + tf.import_cmd(self.tf_import_name, self.tf_import_resource, + capture_output=False, var=self.project_vars()) diff --git a/src/main/python/ddadevops/hetzner_mixin.py b/src/main/python/ddadevops/hetzner_mixin.py index 8993820..014d37b 100644 --- a/src/main/python/ddadevops/hetzner_mixin.py +++ b/src/main/python/ddadevops/hetzner_mixin.py @@ -1,13 +1,22 @@ from .credential import gopass_credential_from_env_path +from .devops_terraform_build import DevopsTerraformBuild -class HetznerMixin: - def __init__(self, project, project_root_path, build_commons_path, module, stage): - super().__init__(self, project, project_root_path, build_commons_path, module, stage) - self.hetzner_api_key = gopass_credential_from_env_path('HETZNER_API_KEY_PATH') +def add_hetzner_mixin_config(config): + return config.update({'HetznerMixin': + {'HETZNER_API_KEY_PATH_ENVIRONMENT': 'HETZNER_API_KEY_PATH'}}) + + +class HetznerMixin(DevopsTerraformBuild): + + def __init__(self, project, config): + super().__init__(self, project, config) + hetzner_mixin_config = config['HetznerMixin'] + self.hetzner_api_key = gopass_credential_from_env_path( + hetzner_mixin_config['HETZNER_API_KEY_PATH_ENVIRONMENT']) def project_vars(self): ret = super().project_vars if self.hetzner_api_key: ret['hetzner_api_key'] = self.hetzner_api_key - return ret \ No newline at end of file + return ret