new layout with base build & mixins

build.py

@@ -27,7 +27,7 @@ use_plugin("python.distutils")
 default_task = "publish"
 
 name = "ddadevops"
-version = "0.3.7.dev4"
+version = "0.4.0.dev1"
 summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
 description = __doc__
 authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]

src/main/python/ddadevops/__init__.py

@@ -4,13 +4,11 @@ terraform, dda-pallet, aws & hetzner-cloud.
 
 """
 
-from .build import init_project, stage, name, module, \
-    project_root_path, build_commons_path, build_target_path, \
-    initialize_target, tf_import_name, tf_import_resource
-from .meissa_build import meissa_init_project, hetzner_api_key
 from .credential import gopass_credential_from_env_path, gopass_credential_from_path
-from .dda_pallet import dda_write_target, dda_write_domain, dda_install, dda_configure
-from .terraform import tf_copy_common, tf_plan, tf_import, tf_apply, tf_output, tf_destroy, \
-    tf_read_output_json
+from .devops_build import DevopsBuild, create_devops_build_config
+from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config
+from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
+from .aws_mixin import AwsMixin, add_aws_mixin_config
+from .dda_pallet_mixin import DdaPalletMixin, add_dda_pallet_mixin_config
 
 __version__ = "${version}"

src/main/python/ddadevops/aws_mixin.py

@@ -0,0 +1,37 @@
+from python_terraform import *
+from .credential import gopass_credential_from_env_path
+from .devops_terraform_build import DevopsTerraformBuild
+
+
+def add_aws_mixin_config(config, account_name):
+    return config.update({'AwsMixin':
+                          {'account_name': account_name}})
+
+
+class AwsMixin(DevopsTerraformBuild):
+
+    def __init__(self, project, config):
+        super().__init__(self, project, config)
+        aws_mixin_config = config['AwsMixin']
+        self.account_name = aws_mixin_config['account_name']
+
+    def backend_config(self):
+        return "backend." + self.account_name + "." + self.stage + ".properties"
+
+    def project_vars(self):
+        ret = super().project_vars()
+        return ret.update({'account_name': self.account_name})
+
+    def init_client(self):
+        tf = Terraform(working_dir=self.build_path())
+        tf.init(backend_config=self.backend_config)
+        try:
+            tf.workspace('select', slef.stage)
+        except:
+            tf.workspace('new', self.stage)
+        return tf
+    
+    def plan(self):
+        tf = self.init_client()
+        tf.plan(capture_output=False, var=self.project_vars,
+                var_file=self.backend_config)

src/main/python/ddadevops/dda_pallet.py

@@ -1,46 +0,0 @@
-from .python_util import *
-from .meissa_build import build_target_path, project_root_path
-from string import Template
-
-
-TARGET = 'target.edn'
-TEMPLATE_TARGET_CONTENT = Template("""
-{:existing [{:node-name "k8s"
-             :node-ip "$ipv4"}]
- :provisioning-user {:login "root"}}
-""")
-
-def dda_write_target(project, ipv4):
-    with open(build_target_path(project) + TARGET, "w") as output_file:
-        output_file.write(TEMPLATE_TARGET_CONTENT.substitute({'ipv4' : ipv4}))
-
-def dda_write_domain(project, domain_file_name, substitues):
-    with open(build_target_path(project) + domain_file_name, "r") as input_file:
-        domain_input = input_file.read()
-    domain_template = Template(domain_input)
-    with open(build_target_path(project) + 'out_' + domain_file_name, "w") as output_file:
-        output_file.write(domain_template.substitute(substitues))
-
-def dda_install(project, tenant, application, domain_file_name):
-    return dda_uberjar(project, tenant, application, domain_file_name)
-
-def dda_configure(project, tenant, application, domain_file_name):
-    return dda_uberjar(project, tenant, application, domain_file_name, True)
-
-def dda_uberjar(project, tenant, application, domain_file_name, configure_switch=None):
-    if configure_switch:
-        cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \
-            '--targets', build_target_path(project) + TARGET, \
-            '--tenant', tenant, '--application', application, \
-            '--configure', \
-            build_target_path(project) + 'out_' + domain_file_name]
-    else:
-        cmd = ['java', '-jar', project_root_path(project) + 'target/meissa-tenant-server.jar', \
-            '--targets', build_target_path(project) + TARGET, \
-            '--tenant', tenant, '--application', application, \
-            build_target_path(project) + 'out_' + domain_file_name]
-    prn_cmd=list(cmd)
-    print(" ".join(prn_cmd))
-    output = execute(cmd)
-    print(output)
-    return output

src/main/python/ddadevops/dda_pallet_mixin.py

@@ -0,0 +1,68 @@
+from string import Template
+from .python_util import *
+from .devops_build import DevopsBuild
+
+
+def add_dda_pallet_mixin_config(config, tenant, application, domain_file_name):
+    return config.update({'DdaPalletMixin':
+                          {'tenant': tenant,
+                           'application': application,
+                           'domain_file_name': domain_file_name,
+                           'target_edn_name': 'target.edn',
+                           'jar_file': 'target/meissa-tenant-server.jar',
+                           'target_template':
+                           """
+{:existing [{:node-name "$node_name"
+             :node-ip "$ipv4"}]
+ :provisioning-user {:login "root"}}
+""", }})
+
+
+class DdaPalletMixin(DevopsBuild):
+
+    def __init__(self, project, config):
+        super().__init__(self, project, config)
+        dda_pallet_mixin_config = config['DdaPalletMixin']
+        self.tenant = dda_pallet_mixin_config['tenant']
+        self.application = dda_pallet_mixin_config['application']
+        self.domain_file_name = dda_pallet_mixin_config['domain_file_name']
+        self.target_edn_name = dda_pallet_mixin_config['target_edn_name']
+        self.jar_file = dda_pallet_mixin_config['jar_file']
+        self.target_template = Template(
+            dda_pallet_mixin_config['target_template'])
+
+    def dda_write_target(self, node_name, ipv4):
+        with open(self.build_path() + self.target_edn_name, "w") as output_file:
+            output_file.write(
+                self.target_template.substitute({'ipv4': ipv4, 'node_name': node_name}))
+
+    def dda_write_domain(self, domain_file_name, substitues):
+        with open(self.build_path() + domain_file_name, "r") as input_file:
+            domain_input = input_file.read()
+        domain_template = Template(domain_input)
+        with open(self.build_path() + 'out_' + domain_file_name, "w") as output_file:
+            output_file.write(domain_template.substitute(substitues))
+
+    def dda_uberjar(self, configure_switch=None):
+        if configure_switch:
+            cmd = ['java', '-jar', self.project_root_path() + self.jar_file,
+                   '--targets', self.build_path() + self.target_edn_name,
+                   '--tenant', self.tenant, '--application', self.application,
+                   '--configure',
+                   self.build_path() + 'out_' + self.domain_file_name]
+        else:
+            cmd = ['java', '-jar', self.project_root_path() + self.jar_file,
+                   '--targets', self.build_path() + self.target_edn_name,
+                   '--tenant', self.tenant, '--application', self.application,
+                   self.build_path() + 'out_' + self.domain_file_name]
+        prn_cmd = list(cmd)
+        print(" ".join(prn_cmd))
+        output = execute(cmd)
+        print(output)
+        return output
+
+    def dda_install(self):
+        return self.dda_uberjar()
+
+    def dda_configure(self):
+        return self.dda_uberjar(True)

src/main/python/ddadevops/devops_build.py

@@ -1,15 +1,23 @@
-from .credential import gopass_credential_from_env_path
 from subprocess import run
 
+
+def create_devops_build_config(stage, project_root_path, build_commons_path, module):
+    return {'stage': stage,
+            'project_root_path': project_root_path,
+            'build_commons_path': build_commons_path,
+            'module': module,
+            'build_dir_name': 'target'}
+
+
 class DevopsBuild:
 
-    def __init__(self, project, project_root_path, build_commons_path, module, stage):
-        self.stage = stage
-        self.project_root_path = project_root_path
-        self.build_commons_path = build_commons_path
-        self.module = module
+    def __init__(self, project, config):
+        self.stage = config['stage']
+        self.project_root_path = config['project_root_path']
+        self.build_commons_path = config['build_commons_path']
+        self.module = config['module']
+        self.build_dir_name = config['build_dir_name']
         self.project = project
-        self.build_dir_name = 'target'
         project.set_property("devops_build", self)
 
     def name(self):
@@ -20,12 +28,12 @@ class DevopsBuild:
 
     def initialize_build_dir(self):
         run('rm -rf ' + self.build_path(), shell=True)
-        run('mkdir -p ' +  self.build_path(), shell=True)
+        run('mkdir -p ' + self.build_path(), shell=True)
 
 
 def tf_import_name(project):
     return project.get_property('tf_import_name')
 
+
 def tf_import_resource(project):
     return project.get_property('tf_import_resource')
-

src/main/python/ddadevops/devops_terraform_build.py

@@ -1,26 +1,37 @@
 from os import path
 from json import load
 from subprocess import run
-from .devops_build import DevopsBuild
-from .python_util import execute
 from python_terraform import *
+from .devops_build import DevopsBuild, create_devops_build_config
+
+
+def create_devops_terraform_build_config(stage, project_root_path, build_commons_path, module,
+                                         account_name, additional_vars, tf_import_name, tf_import_resource):
+    ret = create_devops_build_config(
+        stage, project_root_path, build_commons_path, module)
+    return ret.update({'account_name': account_name,
+                       'additional_vars': additional_vars,
+                       'tf_import_name': tf_import_name,
+                       'tf_import_resource': tf_import_resource,
+                       'terraform_build_commons_dir_name': 'terraform',
+                       'output_json_name': 'output.json'})
+
 
 class DevopsTerraformBuild(DevopsBuild):
 
-    def __init__(self, project, project_root_path, build_commons_path, module, stage, account_name, additional_vars):
-        super().__init__(self, project, project_root_path, build_commons_path, module, stage)
-        self.account_name = account_name
-        self.additional_vars = additional_vars
-        self.terraform_build_commons_dir_name= 'terraform'
-
-    def backend_config(self):
-        return "backend." + self.account_name + "." + self.stage + ".properties"
+    def __init__(self, project, config):
+        super().__init__(self, project, config)
+        self.additional_vars = config['additional_vars']
+        self.tf_import_name = config['tf_import_name']
+        self.tf_import_resource = config['tf_import_resource']
+        self.terraform_build_commons_dir_name = config['terraform_build_commons_dir_name']
+        self.output_json_name = config['output_json_name']
 
     def terraform_build_commons_path(self):
         return self.build_commons_path() + '/' + self.terraform_build_commons_dir_name
 
     def project_vars(self):
-        ret = {'stage' : self.stage}
+        ret = {'stage': self.stage}
         if self.module:
             ret['module'] = self.module
         if self.additional_vars:
@@ -29,77 +40,47 @@ class DevopsTerraformBuild(DevopsBuild):
 
     def initialize_build_dir(self):
         super().initialize_build_dir()
-        run('cp -f ' + self.terraform_build_commons_path + '* ' + self.build_path, shell=True)
-        run('cp *.tf ' +  self.build_path, shell=True)
+        run('cp -f ' + self.terraform_build_commons_path +
+            '* ' + self.build_path, shell=True)
+        run('cp *.tf ' + self.build_path, shell=True)
         run('cp *.properties ' + self.build_path, shell=True)
-        run('cp *.tfars ' +  self.build_path, shell=True)
-        run('cp *.edn ' +  self.build_path, shell=True)
+        run('cp *.tfars ' + self.build_path, shell=True)
+        run('cp *.edn ' + self.build_path, shell=True)
 
     def init_client(self):
         tf = Terraform(working_dir=self.build_path())
-        tf.init(backend_config=self.backend_config)
+        tf.init()
         try:
             tf.workspace('select', slef.stage)
         except:
             tf.workspace('new', self.stage)
         return tf
 
+    def write_output(self, tf):
+        result = tf.output(json=IsFlagged)
+        with open(self.build_path() + self.output_json_name, "w") as output_file:
+            output_file.write(json.dumps(result))
+
+    def read_output_json(self):
+        with open(self.build_path() + self.output_json_name, 'r') as f:
+            return load(f)
+
     def plan(self):
         tf = self.init_client()
-        tf.plan(capture_output=False, var=self.project_vars, var_file=self.backend_config)
+        tf.plan(capture_output=False, var=self.project_vars)
 
+    def apply(self, p_auto_approve=False):
+        tf = self.init_client()
+        tf.apply(capture_output=False, auto_approve=p_auto_approve,
+                 var=self.project_vars())
+        self.write_output(tf)
 
+    def destroy(self, p_auto_approve=False):
+        tf = self.init_client()
+        tf.destroy(capture_output=False, auto_approve=p_auto_approve,
+                   var=self.project_vars())
 
-OUTPUT_JSON = "output.json"
-
-def tf_copy_common(project):
-    run(['cp', '-f', build_commons_path(project) + 'terraform/aws_provider.tf', \
-        build_target_path(project) + 'aws_provider.tf'])
-    run(['cp', '-f', build_commons_path(project) + 'terraform/variables.tf', \
-        build_target_path(project) + 'variables.tf'])
-
-def tf_plan(project):
-    init(project)
-    tf = Terraform(working_dir=build_target_path(project))
-    tf.plan(capture_output=False, var=project_vars(project))
-
-def tf_import(project):
-    init(project)
-    tf = Terraform(working_dir=build_target_path(project))
-    tf.import_cmd(tf_import_name(project), tf_import_resource(project), \
-        capture_output=False, var=project_vars(project))
-
-def tf_apply(project, p_auto_approve=False):
-    init(project)
-    tf = Terraform(working_dir=build_target_path(project))
-    tf.apply(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project))
-    tf_output(project)
-
-def tf_output(project):
-    init(project)
-    tf = Terraform(working_dir=build_target_path(project))
-    result = tf.output(json=IsFlagged)
-    with open(build_target_path(project) + OUTPUT_JSON, "w") as output_file:
-        output_file.write(json.dumps(result))
-    
-def tf_destroy(project, p_auto_approve=False):
-    init(project)
-    tf = Terraform(working_dir=build_target_path(project))
-    tf.destroy(capture_output=False, auto_approve=p_auto_approve, var=project_vars(project))
-
-def tf_read_output_json(project):
-    with open(build_target_path(project) + OUTPUT_JSON, 'r') as f:
-        return load(f)
-
-def project_vars(project):
-    my_hetzner_api_key = hetzner_api_key(project)
-    my_module = project.name
-    ret = {'stage' : stage(project)}
-    # TODO: move to meissa specific part
-    if my_hetzner_api_key:
-        ret['hetzner_api_key'] = my_hetzner_api_key
-    if my_module:
-        ret['module'] = my_module
-    return ret
-
-
+    def tf_import(self):
+        tf = self.init_client()
+        tf.import_cmd(self.tf_import_name, self.tf_import_resource,
+                      capture_output=False, var=self.project_vars())

src/main/python/ddadevops/hetzner_mixin.py

@@ -1,10 +1,19 @@
 from .credential import gopass_credential_from_env_path
+from .devops_terraform_build import DevopsTerraformBuild
 
-class HetznerMixin:
 
-    def __init__(self, project, project_root_path, build_commons_path, module, stage):
-            super().__init__(self, project, project_root_path, build_commons_path, module, stage)
-            self.hetzner_api_key = gopass_credential_from_env_path('HETZNER_API_KEY_PATH')
+def add_hetzner_mixin_config(config):
+    return config.update({'HetznerMixin':
+                          {'HETZNER_API_KEY_PATH_ENVIRONMENT': 'HETZNER_API_KEY_PATH'}})
+
+
+class HetznerMixin(DevopsTerraformBuild):
+
+    def __init__(self, project, config):
+        super().__init__(self, project, config)
+        hetzner_mixin_config = config['HetznerMixin']
+        self.hetzner_api_key = gopass_credential_from_env_path(
+            hetzner_mixin_config['HETZNER_API_KEY_PATH_ENVIRONMENT'])
 
     def project_vars(self):
         ret = super().project_vars