Merge branch 'cleanup-refactorings' into 'main'

Clean old and unused files and functions

See merge request domaindrivenarchitecture/dda-devops-build!7
This commit is contained in:
Pat Dyn 2023-04-28 11:40:47 +00:00
commit 383c92bb9d
54 changed files with 2298 additions and 270 deletions

View file

@ -1,10 +1,10 @@
image: "python:3.8" image: "python:3.10"
before_script: before_script:
- python --version - python --version
- python -m pip install --upgrade pip - python -m pip install --upgrade pip
- pip install -r requirements.txt - pip install -r requirements.txt
stages: stages:
- lint&test - lint&test
- upload - upload
@ -14,20 +14,20 @@ flake8:
stage: lint&test stage: lint&test
script: script:
- pip install -r dev_requirements.txt - pip install -r dev_requirements.txt
- flake8 --max-line-length=120 --count --select=E9,F63,F7,F82 --show-source --statistics src/main/python/ddadevops/*.py - flake8 --max-line-length=120 --count --select=E9,F63,F7,F82 --show-source --statistics src/main/python/ddadevops/
- flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics src/main/python/ddadevops/*.py - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics src/main/python/ddadevops/
mypy: mypy:
stage: lint&test stage: lint&test
script: script:
- pip install -r dev_requirements.txt - pip install -r dev_requirements.txt
- python -m mypy src/main/python/ddadevops/*.py --ignore-missing-imports - python -m mypy src/main/python/ddadevops/ --ignore-missing-imports
pylint: pylint:
stage: lint&test stage: lint&test
script: script:
- pip install -r dev_requirements.txt - pip install -r dev_requirements.txt
- pylint -d C0301,W0614,R0201,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/*.py - pylint -d C0301,W0614,C0114,C0115,C0116,similarities,W0702,W0702,R0913,R0902,R0914,R1732 src/main/python/ddadevops/
pytest: pytest:
stage: lint&test stage: lint&test

View file

@ -165,7 +165,7 @@ def access(project):
build.get_mfa_session() build.get_mfa_session()
``` ```
## Feature DdaDockerBuild ## Feature DdaImageBuild
The docker build supports image building, tagging, testing and login to dockerhost. The docker build supports image building, tagging, testing and login to dockerhost.
For bash based builds we support often used script-parts as predefined functions [see install_functions.sh](src/main/resources/docker/image/resources/install_functions.sh). For bash based builds we support often used script-parts as predefined functions [see install_functions.sh](src/main/resources/docker/image/resources/install_functions.sh).

View file

@ -28,12 +28,12 @@ use_plugin("python.distutils")
default_task = "publish" default_task = "publish"
name = "ddadevops" name = "ddadevops"
version = "3.1.3" version = "4.0.0-dev16"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__ description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
url = "https://github.com/DomainDrivenArchitecture/dda-devops-build" url = "https://github.com/DomainDrivenArchitecture/dda-devops-build"
requires_python = ">=2.7,!=3.0,!=3.1,!=3.2,!=3.3,!=3.4" # CHECK IF NEW VERSION EXISTS requires_python = ">=3.8" # CHECK IF NEW VERSION EXISTS
license = "Apache Software License" license = "Apache Software License"
@init @init
@ -43,7 +43,7 @@ def initialize(project):
project.set_property("verbose", True) project.set_property("verbose", True)
project.get_property("filter_resources_glob").append("main/python/ddadevops/__init__.py") project.get_property("filter_resources_glob").append("main/python/ddadevops/__init__.py")
#project.set_property("dir_source_unittest_python", "src/unittest/python") project.set_property("dir_source_unittest_python", "src/test/python")
project.set_property("copy_resources_target", "$dir_dist/ddadevops") project.set_property("copy_resources_target", "$dir_dist/ddadevops")
project.get_property("copy_resources_glob").append("LICENSE") project.get_property("copy_resources_glob").append("LICENSE")
@ -60,12 +60,9 @@ def initialize(project):
project.set_property("distutils_classifiers", [ project.set_property("distutils_classifiers", [
'License :: OSI Approved :: Apache Software License', 'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python', 'Programming Language :: Python',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.10',
'Operating System :: POSIX :: Linux', 'Operating System :: POSIX :: Linux',
'Operating System :: OS Independent', 'Operating System :: OS Independent',
'Development Status :: 5 - Production/Stable', 'Development Status :: 5 - Production/Stable',

View file

@ -0,0 +1,20 @@
# Architecture
```mermaid
C4Context
title Architectrue od dda-devops-build
Component(buildAndMixin, "Build and Mixin", "")
Component(app, "Application", "")
Component(dom, "Domain", "")
Component(infra, "Infrastructure", "")
Rel(buildAndMixin,app, "use")
Rel(buildAndMixin,dom, "use")
Rel(app, dom, "use")
Rel(app, infra, "use")
UpdateLayoutConfig($c4ShapeInRow="2", $c4BoundaryInRow="1")
```

View file

@ -0,0 +1,104 @@
# Overview of Build and Mixins
* Build can be used standalone
* Mixin can be added to Build
```mermaid
classDiagram
class DevopsBuild {
name()
build_path()
initialize_build_dir()
}
class DevopsTerraformBuild {
terraform_build_commons_path()
project_vars()
copy_build_resource_file_from_package(name)
copy_build_resources_from_package()
copy_build_resources_from_dir()
copy_local_state()
rescue_local_state()
initialize_build_dir()
post_build()
init_client()
write_output(terraform)
read_output_json()
plan()
plan_fail_on_diff()
apply(auto_approve=False)
refresh()
destroy(auto_approve=False)
tf_import(tf_import_name, tf_import_resource,)
print_terraform_command(terraform)
}
class HetznerMixin {
// HetznerMixin -> HetznerTerraformBuild
project_vars()
copy_build_resources_from_package()
}
class ExoscaleMixin {
// ExoscaleMixin -> ExoscaleTerraformBuild
project_vars()
copy_build_resources_from_package()
}
class AwsBackendPropertiesMixin {
def project_vars()
copy_build_resources_from_package()
init_client()
}
class DigitaloceanBackendPropertiesMixin {
project_vars(self)
copy_build_resources_from_package(self)
init_client(self)
}
class DevopsImageBuild {
def initialize_build_dir()
image()
drun()
dockerhub_login()
dockerhub_publish()
test()
}
class ReleaseMixin {
prepare_release()
tag_and_push_release()
}
class ProvsK3sMixin {
// ProvsK3sMixin -> ProvsK3sBuild
def update_runtime_config(fqdn, ipv4, ipv6=None)
write_provs_config()
provs_apply(dry_run=False)
}
class C4kMixin {
// C4kMixin -> C4k
def write_c4k_config()
def write_c4k_auth()
c4k_apply(dry_run=False)
}
DevopsBuild <|-- DevopsImageBuild
DevopsBuild <|-- DevopsTerraformBuild
DevopsBuild <|-- AwsRdsPgMixin
DevopsBuild <|-- ReleaseMixin
DevopsTerraformBuild <|-- AwsBackendPropertiesMixin
DevopsTerraformBuild <|-- DigitaloceanTerraformBuild
DevopsTerraformBuild <|--ExoscaleMixin
DevopsTerraformBuild <|--HetznerMixin
DevopsBuild <|-- ProvsK3sMixin
DigitaloceanTerraformBuild <|-- DigitaloceanBackendPropertiesMixin
AwsBackendPropertiesMixin <|-- AwsMfaMixin
DevopsBuild <|-- C4kMixin
```

View file

@ -0,0 +1,54 @@
# Devops Frontend with application and domain
```mermaid
classDiagram
class DevopsBuild {
__init__(project, config)
do_sth(project)
}
class ProjectRepository {
get_devops(project): Devops
set_devops(project, build)
}
class Devops
class BuildService {
do_sth(project, build)
}
DevopsBuild *-- BuildService
BuildService *-- ProjectRepository
DevopsBuild *-- ProjectRepository
```
In case of simple operations we will not need the BuildService in between.
## Init Sequence
```mermaid
sequenceDiagram
MyBuild ->> DevOpsBuild: create_config
MyBuild ->> DevOpsBuild: __init__(project, config)
activate DevOpsBuild
DevOpsBuild ->> Devops: __init__
DevOpsBuild ->> ProjectRepository: set_devops(build)
deactivate DevOpsBuild
```
## do_sth Sequence
```mermaid
sequenceDiagram
MyBuild ->> DevOpsBuild: do_sth(project)
activate DevOpsBuild
DevOpsBuild ->> BuildService: do_sth(project)
activate BuildService
BuildService ->> ProjectRepository: get_devops
BuildService ->> BuildService: do_some_complicated_stuff(build)
deactivate BuildService
deactivate DevOpsBuild
```

View file

@ -0,0 +1,60 @@
# Domain
```mermaid
classDiagram
class Devops {
stage
name
project_root_path
module
build_dir_name
}
class Image {
dockerhub_user
dockerhub_password
build_dir_name
use_package_common_files
build_commons_path
docker_build_commons_dir_name
docker_publish_tag
}
class C4k {
executabel_name
c4k_mixin_config
c4k_mixin_auth
}
class DnsRecord {
fqdn
ipv4
ipv6
}
class Release {
main_branch
config_file
}
class ReleaseContext {
release_type
version
current_branch
}
C4k *-- DnsRecord
Image *-- Devops
Release *-- "0..1" ReleaseContext
```
# Infrastructure
```mermaid
classDiagram
class ProjectRepository {
get_devops(project): Devops
set_devops(project, build)
}
```

View file

@ -0,0 +1,81 @@
# Architecture of ReleaseMixin
[Link to live editor](https://mermaid.live/edit#pako:eNrtV99vmzAQ_lcsPzUSrUjIj8JDpUqb9jSpaqs9TJGQg6_UGxhmTNes6v8-E0MCGIem6-MiBYnz3Xd3vu8ulxccZRRwgAv4VQKP4BMjsSApUp81r54CIolEvDmbup6D6sdEn21KllB0fnWFbiEBUsBX9sx4gMKQcSbD8CwX2Q9l76Ao4w8sDh9YArUtiSR7IhI6pge3HWnl4QuT1zlrYU8sirXgfpvDLeRZwWQmti07DVRb50RIFrGccNk2tEB_A1GwjA_Cmhm2sWvL4yEP4ho-neEMHZQSxsONIDx6tGenD4BTo7oO8tV3NVLaXIBChVBoaVOFPc7MrfixcNDCtRXoRkPU8jsQTyyCVsbGbfQZMwigdQaPbHccg-zn0WflQb2TzEF8jHIt_FCqM5uTrl3HUfeo0wgVeqJQChlGWZoyacBrTS2kMCi2u2mdBOjQly2c8eh7kAPtUyXxpMVG-Ia6PjfENuwkI3TXjw3ymy0VhVTJ3mza4m5l46A6ozBhhZwY92bJ6yi1zPaort1psNSALYUALrv9v29zs2p972Pn4wgfMAJ-CyYhJJzWjO5352h3B6jpNxupOmOwfunWMhKgFEMD6FgPjIVnHXlGxo27OpzJO8baiS2oQ9iRveFtIQXj8ajvZhIRSs_erNydVeYP0QeyZ1Om-QnUqdSHyn06d2xI_4nzYcRpXeWRdWBPr4GFpyLaym3xzLbySBLvLjovi8fDRDqyqt6T-JrTG6Vu3XE6S-g-E5vhu3wNh61hrI7GIU9BakqnQ-GZVEnSf4zh5HSaICrDAfbYbG0du7v6HqmqJ3ZwCkLt4FT9m3qpJGssHyGFNVadhSkRP9d4zV-VHilldrflEQ4eSFKAg8ucKg_1X6-e9DOt2m0vVLv89yxTSlKU-hUHL_gZB-fT6cWlt_Td5dzz1ACdL30Hb5Xcny4uZjN_7k2ni5k39y5fHfxnBzG7cD135fnzxdJfrObu6vUveFPSvA)
```mermaid
sequenceDiagram
rect rgb(103, 103, 10)
build ->> ReleaseMixin: __init__(project, config_file)
activate ReleaseMixin
ReleaseMixin ->> GitApi: __init__()
ReleaseMixin ->> ReleaseTypeRepository: __init__(GitApi)
participant ReleaseType
ReleaseMixin ->> VersionRepository: __init__(config_file)
participant Version
ReleaseMixin ->> ReleaseRepository: __init__(VersionRepository, ReleaseTypeRepository, main_branch)
participant Release
end
rect rgb(10, 90, 7)
build ->> ReleaseMixin: prepare_release()
rect rgb(20, 105, 50)
ReleaseMixin ->> PrepareReleaseService: __init__(ReleaseRepository)
activate PrepareReleaseService
PrepareReleaseService ->> ReleaseRepository: get_release()
activate ReleaseRepository
ReleaseRepository ->> ReleaseTypeRepository: get_release_type()
activate ReleaseTypeRepository
ReleaseTypeRepository ->> GitApi: get_latest_commit()
activate GitApi
deactivate GitApi
ReleaseTypeRepository ->> ReleaseType:
deactivate ReleaseTypeRepository
ReleaseRepository ->> VersionRepository: get_version()
activate VersionRepository
VersionRepository ->> VersionRepository: load_file()
VersionRepository ->> VersionRepository: parse_file()
VersionRepository ->> Version: __init__(file, version_list)
deactivate VersionRepository
ReleaseRepository ->> Release: __init__(ReleaseType, Version, current_branch)
end
deactivate ReleaseRepository
activate ReleaseRepository
deactivate ReleaseRepository
rect rgb(20, 105, 50)
ReleaseMixin ->> PrepareReleaseService: write_and_commit_release()
PrepareReleaseService ->> Release: release_version()
activate Release
Release ->> Version: create_release_version()
deactivate Release
PrepareReleaseService ->> PrepareReleaseService: __write_and_commit_version(Version)
PrepareReleaseService ->> ReleaseRepository:
ReleaseRepository ->> VersionRepository: write_file(version_string)
PrepareReleaseService ->> GitApi: add()
PrepareReleaseService ->> GitApi: commit()
end
rect rgb(20, 105, 50)
ReleaseMixin ->> PrepareReleaseService: write_and_commit_bump()
PrepareReleaseService ->> Release: bump_version()
activate Release
Release ->> Version: create_bump_version()
deactivate Release
PrepareReleaseService ->> PrepareReleaseService: __write_and_commit_version(Version)
PrepareReleaseService ->> ReleaseRepository:
ReleaseRepository ->> VersionRepository: write_file(version_string)
PrepareReleaseService ->> GitApi: add()
PrepareReleaseService ->> GitApi: commit()
deactivate PrepareReleaseService
end
end
rect rgb(120, 70, 50)
build ->> ReleaseMixin: tag_and_push_release()
ReleaseMixin ->> TagAndPushReleaseService: __init__(GitApi)
activate TagAndPushReleaseService
ReleaseMixin ->> TagAndPushReleaseService: tag_and_push_release()
TagAndPushReleaseService ->> TagAndPushReleaseService: tag_release()
TagAndPushReleaseService ->> GitApi: tag_annotated()
TagAndPushReleaseService ->> TagAndPushReleaseService: push_release()
TagAndPushReleaseService ->> GitApi: push()
deactivate TagAndPushReleaseService
deactivate ReleaseMixin
end
```

View file

@ -1,6 +1,15 @@
# For local development
``` ```
python3 -m venv ~/.venv --upgrade python3 -m venv ~/.venv --upgrade
source ~/.venv/bin/activate source ~/.venv/bin/activate
pip3 install --upgrade pybuilder deprecation dda_python_terraform boto3 pip3 install --upgrade -r dev_requirements.txt
pip3 install --upgrade -r requirements.txt
```
# For testing a dev version
```
pyb publish upload
pip3 install --upgrade ddadevops --pre pip3 install --upgrade ddadevops --pre
``` ```

View file

@ -5,7 +5,7 @@ name = 'example-project'
MODULE = 'docker-module' MODULE = 'docker-module'
PROJECT_ROOT_PATH = '../../..' PROJECT_ROOT_PATH = '../../..'
class MyBuild(DevopsDockerBuild): class MyBuild(DevopsImageBuild):
pass pass
@init @init

View file

@ -1,33 +1,40 @@
from subprocess import run
from os import environ from os import environ
from pybuilder.core import task, init from pybuilder.core import task, init
from ddadevops import * from ddadevops import *
import logging
name = 'clojure' name = "clojure"
MODULE = 'docker' MODULE = "docker"
PROJECT_ROOT_PATH = '../..' PROJECT_ROOT_PATH = "../.."
class MyBuild(DevopsDockerBuild):
pass
@init @init
def initialize(project): def initialize(project):
project.build_depends_on('ddadevops>=0.13.0') project.build_depends_on("ddadevops>=4.0.0-dev")
stage = 'notused' stage = "notused"
dockerhub_user = environ.get('DOCKERHUB_USER') dockerhub_user = environ.get("DOCKERHUB_USER")
if not dockerhub_user: if not dockerhub_user:
dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') dockerhub_user = gopass_field_from_path("meissa/web/docker.com", "login")
dockerhub_password = environ.get('DOCKERHUB_PASSWORD') dockerhub_password = environ.get("DOCKERHUB_PASSWORD")
if not dockerhub_password: if not dockerhub_password:
dockerhub_password = gopass_password_from_path('meissa/web/docker.com') dockerhub_password = gopass_password_from_path("meissa/web/docker.com")
tag = environ.get('CI_COMMIT_TAG') tag = environ.get("CI_COMMIT_TAG")
if not tag: if not tag:
tag = get_tag_from_latest_commit() tag = get_tag_from_latest_commit()
config = create_devops_docker_build_config(
stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) devops = Devops(
build = MyBuild(project, config) stage=stage,
project_root_path=PROJECT_ROOT_PATH,
module=MODULE,
name=name,
)
image = Image(
dockerhub_user=dockerhub_user,
dockerhub_password=dockerhub_password,
docker_publish_tag=tag,
devops=devops,
)
build = DevopsImageBuild(project, image=image)
build.initialize_build_dir() build.initialize_build_dir()
@ -36,16 +43,19 @@ def image(project):
build = get_devops_build(project) build = get_devops_build(project)
build.image() build.image()
@task @task
def drun(project): def drun(project):
build = get_devops_build(project) build = get_devops_build(project)
build.drun() build.drun()
@task @task
def test(project): def test(project):
build = get_devops_build(project) build = get_devops_build(project)
build.test() build.test()
@task @task
def publish(project): def publish(project):
build = get_devops_build(project) build = get_devops_build(project)

View file

@ -1,4 +1,4 @@
FROM domaindrivenarchitecture/clojure FROM clojure
RUN apt update RUN apt update
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless

View file

@ -1,33 +1,40 @@
from subprocess import run
from os import environ from os import environ
from pybuilder.core import task, init from pybuilder.core import task, init
from ddadevops import * from ddadevops import *
import logging
name = 'devops-build' name = "devops-build"
MODULE = 'docker' MODULE = "docker"
PROJECT_ROOT_PATH = '../..' PROJECT_ROOT_PATH = "../.."
class MyBuild(DevopsDockerBuild):
pass
@init @init
def initialize(project): def initialize(project):
project.build_depends_on('ddadevops>=0.13.0') project.build_depends_on("ddadevops>=4.0.0-dev")
stage = 'notused' stage = "notused"
dockerhub_user = environ.get('DOCKERHUB_USER') dockerhub_user = environ.get("DOCKERHUB_USER")
if not dockerhub_user: if not dockerhub_user:
dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login') dockerhub_user = gopass_field_from_path("meissa/web/docker.com", "login")
dockerhub_password = environ.get('DOCKERHUB_PASSWORD') dockerhub_password = environ.get("DOCKERHUB_PASSWORD")
if not dockerhub_password: if not dockerhub_password:
dockerhub_password = gopass_password_from_path('meissa/web/docker.com') dockerhub_password = gopass_password_from_path("meissa/web/docker.com")
tag = environ.get('CI_COMMIT_TAG') tag = environ.get("CI_COMMIT_TAG")
if not tag: if not tag:
tag = get_tag_from_latest_commit() tag = get_tag_from_latest_commit()
config = create_devops_docker_build_config(
stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag) devops = Devops(
build = MyBuild(project, config) stage=stage,
project_root_path=PROJECT_ROOT_PATH,
module=MODULE,
name=name,
)
image = Image(
dockerhub_user=dockerhub_user,
dockerhub_password=dockerhub_password,
docker_publish_tag=tag,
devops=devops,
)
build = DevopsImageBuild(project, image=image)
build.initialize_build_dir() build.initialize_build_dir()
@ -36,16 +43,19 @@ def image(project):
build = get_devops_build(project) build = get_devops_build(project)
build.image() build.image()
@task @task
def drun(project): def drun(project):
build = get_devops_build(project) build = get_devops_build(project)
build.drun() build.drun()
@task @task
def test(project): def test(project):
build = get_devops_build(project) build = get_devops_build(project)
build.test() build.test()
@task @task
def publish(project): def publish(project):
build = get_devops_build(project) build = get_devops_build(project)

View file

@ -1,7 +1,7 @@
FROM docker:latest FROM docker:latest
RUN set -eux; RUN set -eux;
RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptools py3-wheel libffi-dev openssl-dev cargo; RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptools py3-wheel libffi-dev openssl-dev cargo bash;
RUN python3 -m pip install -U pip; RUN python3 -m pip install -U pip;
RUN ln -s /usr/bin/python3 /usr/bin/python #RUN ln -s /usr/bin/python3 /usr/bin/python
RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 mfa; RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 mfa;

View file

@ -1,4 +1,4 @@
FROM domaindrivenarchitecture/devops-build FROM devops-build
RUN apk update RUN apk update
RUN apk add curl openjdk8 RUN apk add curl openjdk8

View file

@ -9,14 +9,17 @@ from .provs_k3s_mixin import ProvsK3sMixin, add_provs_k3s_mixin_config
from .aws_rds_pg_mixin import AwsRdsPgMixin, add_aws_rds_pg_mixin_config from .aws_rds_pg_mixin import AwsRdsPgMixin, add_aws_rds_pg_mixin_config
from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config from .aws_mfa_mixin import AwsMfaMixin, add_aws_mfa_mixin_config
from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config from .aws_backend_properties_mixin import AwsBackendPropertiesMixin, add_aws_backend_properties_mixin_config
from .c4k_mixin import C4kMixin, add_c4k_mixin_config from .c4k_mixin import C4kBuild, add_c4k_mixin_config
from .exoscale_mixin import ExoscaleMixin, add_exoscale_mixin_config from .exoscale_mixin import ExoscaleMixin, add_exoscale_mixin_config
from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config
from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config
from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
from .devops_docker_build import DevopsDockerBuild, create_devops_docker_build_config from .devops_image_build import DevopsImageBuild, create_devops_docker_build_config
from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config
from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build, get_tag_from_latest_commit from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build
from .credential import gopass_password_from_path, gopass_field_from_path from .credential import gopass_password_from_path, gopass_field_from_path
from .release_mixin import ReleaseMixin
from .domain import Validateable, DnsRecord, Devops, Image, Release, ReleaseContext
__version__ = "${version}" __version__ = "${version}"

View file

@ -0,0 +1,2 @@
from .image_build_service import ImageBuildService
from .release_mixin_services import TagAndPushReleaseService, PrepareReleaseService

View file

@ -0,0 +1,54 @@
from src.main.python.ddadevops.domain import Image
from src.main.python.ddadevops.infrastructure import FileApi, ResourceApi, ImageApi
class ImageBuildService:
def __init__(self):
self.file_api = FileApi()
self.resource_api = ResourceApi()
self.docker_api = ImageApi()
def __copy_build_resource_file_from_package__(self, resource_name, docker: Image):
data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}")
self.file_api.write_data_to_file(
f"{docker.devops.build_path()}/{resource_name}", data
)
def __copy_build_resources_from_package__(self, docker: Image):
self.__copy_build_resource_file_from_package__(
"image/resources/install_functions.sh", docker
)
def __copy_build_resources_from_dir__(self, docker: Image):
self.file_api.cp_force(
docker.docker_build_commons_path(), docker.devops.build_path()
)
def initialize_build_dir(self, docker: Image):
build_path = docker.devops.build_path()
self.file_api.clean_dir(f"{build_path}/image/resources")
if docker.use_package_common_files:
self.__copy_build_resources_from_package__(docker)
else:
self.__copy_build_resources_from_dir__(docker)
self.file_api.cp_recursive("image", build_path)
self.file_api.cp_recursive("test", build_path)
def image(self, docker: Image):
self.docker_api.image(docker.devops.name, docker.devops.build_path())
def drun(self, docker: Image):
self.docker_api.drun(docker.devops.name)
def dockerhub_login(self, docker: Image):
self.docker_api.dockerhub_login(
docker.dockerhub_user, docker.dockerhub_password
)
def dockerhub_publish(self, docker: Image):
self.docker_api.dockerhub_publish(
docker.devops.name, docker.dockerhub_user, docker.docker_publish_tag
)
def test(self, docker: Image):
self.docker_api.test(docker.devops.name, docker.devops.build_path())

View file

@ -0,0 +1,34 @@
from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, VersionRepository, GitApi
from src.main.python.ddadevops.domain import Version, Release
class PrepareReleaseService():
def __init__(self):
self.git_api = GitApi()
def __write_and_commit_version(self, release: Release, version_repository: VersionRepository, version: Version, commit_message: str):
release.is_valid()
version_repository.write_file(version.get_version_string())
self.git_api.add_file(version_repository.file)
self.git_api.commit(commit_message)
def write_and_commit_release(self, release: Release, version_repository: VersionRepository):
self.__write_and_commit_version(release, version_repository, release.release_version(), commit_message=f'Release v{release.release_version().get_version_string()}')
def write_and_commit_bump(self, release: Release, version_repository: VersionRepository):
self.__write_and_commit_version(release, version_repository, release.bump_version(), commit_message='Version bump')
class TagAndPushReleaseService():
def __init__(self, git_api: GitApi):
self.git_api = git_api
def tag_release(self, release_repo: ReleaseContextRepository):
annotation = 'v' + release_repo.get_release().version.get_version_string()
message = 'Release ' + annotation
self.git_api.tag_annotated_second_last(annotation, message)
def push_release(self):
self.git_api.push()

View file

@ -1,63 +1,74 @@
from os import chmod import deprecation
import yaml from .domain import C4k, DnsRecord
from .python_util import execute
from .devops_build import DevopsBuild from .devops_build import DevopsBuild
from .credential import gopass_field_from_path, gopass_password_from_path from .credential import gopass_field_from_path, gopass_password_from_path
from .infrastructure import ExecutionApi
def add_c4k_mixin_config(config,
c4k_module_name, @deprecation.deprecated(deprecated_in="3.2")
c4k_config_dict, # create objects direct instead
c4k_auth_dict, def add_c4k_mixin_config(
grafana_cloud_user=None, config,
grafana_cloud_password=None, c4k_config_dict,
grafana_cloud_url='https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push'): c4k_auth_dict,
executabel_name=None,
grafana_cloud_user=None,
grafana_cloud_password=None,
grafana_cloud_url="https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push",
):
if not grafana_cloud_user: if not grafana_cloud_user:
grafana_cloud_user = gopass_field_from_path( grafana_cloud_user = gopass_field_from_path(
'server/meissa/grafana-cloud', 'grafana-cloud-user') "server/meissa/grafana-cloud", "grafana-cloud-user"
)
if not grafana_cloud_password: if not grafana_cloud_password:
grafana_cloud_password = gopass_password_from_path( grafana_cloud_password = gopass_password_from_path(
'server/meissa/grafana-cloud') "server/meissa/grafana-cloud"
c4k_auth_dict.update({'mon-auth': { )
'grafana-cloud-user': grafana_cloud_user, c4k_auth_dict.update(
'grafana-cloud-password': grafana_cloud_password {
}}) "mon-auth": {
c4k_config_dict.update({'mon-cfg': { "grafana-cloud-user": grafana_cloud_user,
'grafana-cloud-url': grafana_cloud_url "grafana-cloud-password": grafana_cloud_password,
}}) }
config.update({'C4kMixin': {'Config': c4k_config_dict, }
'Auth': c4k_auth_dict, )
'Name': c4k_module_name}}) c4k_config_dict.update({"mon-cfg": {"grafana-cloud-url": grafana_cloud_url}})
config.update(
{
"C4kMixin": {
"executabel_name": executabel_name,
"config": c4k_config_dict,
"auth": c4k_auth_dict,
}
}
)
return config return config
class C4kBuild(DevopsBuild):
class C4kMixin(DevopsBuild):
def __init__(self, project, config): def __init__(self, project, config):
super().__init__(project, config) super().__init__(project, config)
self.c4k_mixin_config = config['C4kMixin']['Config'] self.execution_api = ExecutionApi()
self.c4k_mixin_auth = config['C4kMixin']['Auth'] c4k_build = C4k(config)
self.c4k_module_name = config['C4kMixin']['Name'] self.repo.set_c4k(self.project, c4k_build)
tmp = self.c4k_mixin_config['mon-cfg']
tmp.update({'cluster-name': self.c4k_module_name, def update_runtime_config(self, dns_record: DnsRecord):
'cluster-stage': self.stage}) c4k_build = self.repo.get_c4k(self.project)
self.c4k_mixin_config.update({'mon-cfg': tmp}) c4k_build.update_runtime_config(dns_record)
self.repo.set_c4k(self.project, c4k_build)
def write_c4k_config(self): def write_c4k_config(self):
fqdn = self.get('fqdn') build = self.repo.get_devops(self.project)
self.c4k_mixin_config.update({'fqdn': fqdn}) c4k_build = self.repo.get_c4k(self.project)
with open(self.build_path() + '/out_c4k_config.yaml', 'w', encoding="utf-8") as output_file: path = build.build_path() + "/out_c4k_config.yaml"
yaml.dump(self.c4k_mixin_config, output_file) self.file_api.write_yaml_to_file(path, c4k_build.config())
def write_c4k_auth(self): def write_c4k_auth(self):
with open(self.build_path() + '/out_c4k_auth.yaml', 'w', encoding="utf-8") as output_file: build = self.repo.get_devops(self.project)
yaml.dump(self.c4k_mixin_auth, output_file) c4k_build = self.repo.get_c4k(self.project)
chmod(self.build_path() + '/out_c4k_auth.yaml', 0o600) path = build.build_path() + "/out_c4k_auth.yaml"
self.file_api.write_yaml_to_file(path, c4k_build.c4k_mixin_auth)
def c4k_apply(self, dry_run=False): def c4k_apply(self, dry_run=False):
cmd = f'c4k-{self.c4k_module_name}-standalone.jar {self.build_path()}/out_c4k_config.yaml {self.build_path()}/out_c4k_auth.yaml > {self.build_path()}/out_{self.c4k_module_name}.yaml' build = self.repo.get_devops(self.project)
output = '' c4k_build = self.repo.get_c4k(self.project)
if dry_run: return self.execution_api.execute(c4k_build.command(build), dry_run)
print(cmd)
else:
output = execute(cmd, True)
print(output)
return output

View file

@ -1,58 +1,51 @@
from typing import Optional
from subprocess import run, CalledProcessError from subprocess import run, CalledProcessError
from .python_util import filter_none import deprecation
from .domain import Devops
from .infrastructure import ProjectRepository, FileApi
def create_devops_build_config(stage, project_root_path, module,
build_dir_name='target'): @deprecation.deprecated(deprecated_in="3.2", details="create objects direct instead")
return {'stage': stage, def create_devops_build_config(
'project_root_path': project_root_path, stage, project_root_path, module, build_dir_name="target"
'module': module, ):
'build_dir_name': build_dir_name} return {
"stage": stage,
"project_root_path": project_root_path,
"module": module,
"build_dir_name": build_dir_name,
}
def get_devops_build(project): def get_devops_build(project):
return project.get_property('devops_build') return project.get_property("devops_build")
def get_tag_from_latest_commit():
try:
value = run('git describe --abbrev=0 --tags --exact-match', shell=True,
capture_output=True, check=True)
return value.stdout.decode('UTF-8').rstrip()
except CalledProcessError:
return None
class DevopsBuild: class DevopsBuild:
def __init__(self, project, config: Optional[dict] = None, devops: Optional[Devops] = None):
def __init__(self, project, config):
#deprecate stage
self.stage = config['stage']
self.project_root_path = config['project_root_path']
self.module = config['module']
self.build_dir_name = config['build_dir_name']
self.stack = {}
self.project = project self.project = project
project.set_property('devops_build', self) self.file_api = FileApi()
self.repo = ProjectRepository()
if not devops:
if not config:
raise ValueError("Build parameters could not be set!")
devops = Devops(
stage=config["stage"],
project_root_path=config["project_root_path"],
module=config["module"],
name=project.name,
build_dir_name=config["build_dir_name"],
)
self.repo.set_devops(self.project, devops)
self.repo.set_build(self.project, self)
def name(self): def name(self):
return self.project.name devops = self.repo.get_devops(self.project)
return devops.name
def build_path(self): def build_path(self):
mylist = [self.project_root_path, devops = self.repo.get_devops(self.project)
self.build_dir_name, return devops.build_path()
self.name(),
self.module]
return '/'.join(filter_none(mylist))
def initialize_build_dir(self): def initialize_build_dir(self):
run('rm -rf ' + self.build_path(), shell=True, check=True) devops = self.repo.get_devops(self.project)
run('mkdir -p ' + self.build_path(), shell=True, check=True) self.file_api.clean_dir(devops.build_path())
def put(self, key, value):
self.stack[key] = value
def get(self, key):
return self.stack[key]
def get_keys(self, keys):
result = {}
for key in keys:
result[key] = self.get(key)
return result

View file

@ -1,96 +0,0 @@
import sys
from subprocess import run
from pkg_resources import resource_string
from .python_util import filter_none
from .devops_build import DevopsBuild, create_devops_build_config
def create_devops_docker_build_config(stage,
project_root_path,
module,
dockerhub_user,
dockerhub_password,
build_dir_name='target',
use_package_common_files=True,
build_commons_path=None,
docker_build_commons_dir_name='docker',
docker_publish_tag=None):
ret = create_devops_build_config(
stage, project_root_path, module, build_dir_name)
ret.update({'dockerhub_user': dockerhub_user,
'dockerhub_password': dockerhub_password,
'use_package_common_files': use_package_common_files,
'docker_build_commons_dir_name': docker_build_commons_dir_name,
'build_commons_path': build_commons_path,
'docker_publish_tag': docker_publish_tag, })
return ret
class DevopsDockerBuild(DevopsBuild):
def __init__(self, project, config):
super().__init__(project, config)
project.build_depends_on('dda-python-terraform')
self.dockerhub_user = config['dockerhub_user']
self.dockerhub_password = config['dockerhub_password']
self.use_package_common_files = config['use_package_common_files']
self.build_commons_path = config['build_commons_path']
self.docker_build_commons_dir_name = config['docker_build_commons_dir_name']
self.docker_publish_tag = config['docker_publish_tag']
def docker_build_commons_path(self):
mylist = [self.build_commons_path,
self.docker_build_commons_dir_name]
return '/'.join(filter_none(mylist)) + '/'
def copy_build_resource_file_from_package(self, name):
run('mkdir -p ' + self.build_path() + '/image/resources', shell=True, check=True)
my_data = resource_string(
__name__, "src/main/resources/docker/" + name)
with open(self.build_path() + '/' + name, "w", encoding="utf-8") as output_file:
output_file.write(my_data.decode(sys.stdout.encoding))
def copy_build_resources_from_package(self):
self.copy_build_resource_file_from_package(
'image/resources/install_functions.sh')
def copy_build_resources_from_dir(self):
run('cp -f ' + self.docker_build_commons_path() +
'* ' + self.build_path(), shell=True, check=True)
def initialize_build_dir(self):
super().initialize_build_dir()
if self.use_package_common_files:
self.copy_build_resources_from_package()
else:
self.copy_build_resources_from_dir()
run('cp -r image ' + self.build_path(), shell=True, check=True)
run('cp -r test ' + self.build_path(), shell=True, check=True)
def image(self):
run('docker build -t ' + self.name() +
' --file ' + self.build_path() + '/image/Dockerfile '
+ self.build_path() + '/image', shell=True, check=True)
def drun(self):
run('docker run --expose 8080 -it --entrypoint="" ' +
self.name() + ' /bin/bash', shell=True, check=True)
def dockerhub_login(self):
run('docker login --username ' + self.dockerhub_user +
' --password ' + self.dockerhub_password, shell=True, check=True)
def dockerhub_publish(self):
if self.docker_publish_tag is not None:
run('docker tag ' + self.name() + ' ' + self.dockerhub_user +
'/' + self.name() + ':' + self.docker_publish_tag, shell=True, check=True)
run('docker push ' + self.dockerhub_user +
'/' + self.name() + ':' + self.docker_publish_tag, shell=True, check=True)
run('docker tag ' + self.name() + ' ' + self.dockerhub_user +
'/' + self.name() + ':latest', shell=True, check=True)
run('docker push ' + self.dockerhub_user +
'/' + self.name() + ':latest', shell=True, check=True)
def test(self):
run('docker build -t ' + self.name() + '-test ' +
'--file ' + self.build_path() + '/test/Dockerfile '
+ self.build_path() + '/test', shell=True, check=True)

View file

@ -0,0 +1,78 @@
from typing import Optional
import deprecation
from .domain import Image
from .application import ImageBuildService
from .devops_build import DevopsBuild, create_devops_build_config
@deprecation.deprecated(deprecated_in="3.2", details="create objects direct instead")
def create_devops_docker_build_config(
stage,
project_root_path,
module,
dockerhub_user,
dockerhub_password,
build_dir_name="target",
use_package_common_files=True,
build_commons_path=None,
docker_build_commons_dir_name="docker",
docker_publish_tag=None,
):
ret = create_devops_build_config(stage, project_root_path, module, build_dir_name)
ret.update(
{
"dockerhub_user": dockerhub_user,
"dockerhub_password": dockerhub_password,
"use_package_common_files": use_package_common_files,
"docker_build_commons_dir_name": docker_build_commons_dir_name,
"build_commons_path": build_commons_path,
"docker_publish_tag": docker_publish_tag,
}
)
return ret
class DevopsImageBuild(DevopsBuild):
def __init__(self, project, config: Optional[dict] = None, image: Optional[Image] = None):
self.image_build_service = ImageBuildService()
if not image:
if not config:
raise ValueError("Image parameters could not be set.")
super().__init__(project, config=config)
image = Image(
dockerhub_user=config["dockerhub_user"],
dockerhub_password=config["dockerhub_password"],
devops=self.repo.get_devops(project),
use_package_common_files=config["use_package_common_files"],
build_commons_path=config["build_commons_path"],
docker_build_commons_dir_name=config["docker_build_commons_dir_name"],
docker_publish_tag=config["docker_publish_tag"],
)
else:
super().__init__(project, devops=image.devops)
self.repo.set_docker(self.project, image)
def initialize_build_dir(self):
super().initialize_build_dir()
image = self.repo.get_docker(self.project)
self.image_build_service.initialize_build_dir(image)
def image(self):
image = self.repo.get_docker(self.project)
self.image_build_service.image(image)
def drun(self):
image = self.repo.get_docker(self.project)
self.image_build_service.drun(image)
def dockerhub_login(self):
image = self.repo.get_docker(self.project)
self.image_build_service.dockerhub_login(image)
def dockerhub_publish(self):
image = self.repo.get_docker(self.project)
self.image_build_service.dockerhub_publish(image)
def test(self):
image = self.repo.get_docker(self.project)
self.image_build_service.test(image)

View file

@ -55,6 +55,8 @@ class DevopsTerraformBuild(DevopsBuild):
self.debug_print_terraform_command = config['debug_print_terraform_command'] self.debug_print_terraform_command = config['debug_print_terraform_command']
self.additional_tfvar_files = config['additional_tfvar_files'] self.additional_tfvar_files = config['additional_tfvar_files']
self.terraform_semantic_version = config['terraform_semantic_version'] self.terraform_semantic_version = config['terraform_semantic_version']
self.stage = config["stage"]
self.module = config["module"]
def terraform_build_commons_path(self): def terraform_build_commons_path(self):
mylist = [self.build_commons_path, mylist = [self.build_commons_path,
@ -136,7 +138,7 @@ class DevopsTerraformBuild(DevopsBuild):
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
raise Exception(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
def plan_fail_on_diff(self): def plan_fail_on_diff(self):
terraform = self.init_client() terraform = self.init_client()
@ -146,9 +148,9 @@ class DevopsTerraformBuild(DevopsBuild):
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code not in (0, 2): if return_code not in (0, 2):
raise Exception(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
if return_code == 2: if return_code == 2:
raise Exception(return_code, "diff in config found:", stderr) raise RuntimeError(return_code, "diff in config found:", stderr)
def apply(self, auto_approve=False): def apply(self, auto_approve=False):
terraform = self.init_client() terraform = self.init_client()
@ -170,7 +172,7 @@ class DevopsTerraformBuild(DevopsBuild):
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
raise Exception(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
def refresh(self): def refresh(self):
terraform = self.init_client() terraform = self.init_client()
@ -181,7 +183,7 @@ class DevopsTerraformBuild(DevopsBuild):
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
raise Exception(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
def destroy(self, auto_approve=False): def destroy(self, auto_approve=False):
terraform = self.init_client() terraform = self.init_client()
@ -202,7 +204,7 @@ class DevopsTerraformBuild(DevopsBuild):
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
raise Exception(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
def tf_import(self, tf_import_name, tf_import_resource,): def tf_import(self, tf_import_name, tf_import_resource,):
terraform = self.init_client() terraform = self.init_client()
@ -213,7 +215,7 @@ class DevopsTerraformBuild(DevopsBuild):
self.post_build() self.post_build()
self.print_terraform_command(terraform) self.print_terraform_command(terraform)
if return_code > 0: if return_code > 0:
raise Exception(return_code, "terraform error:", stderr) raise RuntimeError(return_code, "terraform error:", stderr)
def print_terraform_command(self, terraform): def print_terraform_command(self, terraform):
if self.debug_print_terraform_command: if self.debug_print_terraform_command:

View file

@ -0,0 +1,4 @@
from .common import Validateable, DnsRecord, Devops
from .image import Image
from .c4k import C4k
from .release import Release, ReleaseContext, ReleaseType, Version, EnvironmentKeys

View file

@ -0,0 +1,43 @@
from typing import List, Optional
from .common import (
Validateable,
DnsRecord,
Devops,
)
class C4k(Validateable):
def __init__(self, config: dict):
tmp_executabel_name = config["C4kMixin"]["executabel_name"]
if not tmp_executabel_name:
tmp_executabel_name = config["module"]
self.executabel_name = tmp_executabel_name
self.c4k_mixin_config = config["C4kMixin"]["config"]
self.c4k_mixin_auth = config["C4kMixin"]["auth"]
tmp = self.c4k_mixin_config["mon-cfg"]
tmp.update({"cluster-name": config["module"], "cluster-stage": config["stage"]})
self.c4k_mixin_config.update({"mon-cfg": tmp})
self.dns_record: Optional[DnsRecord] = None
# TODO: these functions should be located at TerraformBuild later on.
def update_runtime_config(self, dns_record: DnsRecord):
self.dns_record = dns_record
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("fqdn")
if self.dns_record:
result += self.dns_record.validate()
return result
def config(self):
fqdn = self.dns_record.fqdn
self.c4k_mixin_config.update({"fqdn": fqdn})
return self.c4k_mixin_config
def command(self, build: Devops):
module = build.module
build_path = build.build_path()
config_path = f"{build_path}/out_c4k_config.yaml"
auth_path = f"{build_path}/out_c4k_auth.yaml"
output_path = f"{build_path}/out_{module}.yaml"
return f"c4k-{self.executabel_name}-standalone.jar {config_path} {auth_path} > {output_path}"

View file

@ -0,0 +1,72 @@
import deprecation
import logging
from typing import List
def filter_none(list_to_filter):
return [x for x in list_to_filter if x is not None]
class Validateable:
def __validate_is_not_empty__(self, field_name: str) -> List[str]:
value = self.__dict__[field_name]
if value is None or value == "":
return [f"Field '{field_name}' must not be empty."]
else:
return []
def validate(self) -> List[str]:
return []
def is_valid(self) -> bool:
return len(self.validate()) < 1
class DnsRecord(Validateable):
def __init__(self, fqdn, ipv4=None, ipv6=None):
self.fqdn = fqdn
self.ipv4 = ipv4
self.ipv6 = ipv6
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("fqdn")
if (not self.ipv4) and (not self.ipv6):
result.append("ipv4 & ipv6 may not both be empty.")
return result
class Devops(Validateable):
def __init__(
self, stage: str, project_root_path: str, module: str, name: str | None =None, build_dir_name: str="target"
):
self.stage = stage
self.name = name
self.project_root_path = project_root_path
logging.warn(f"Set project root in DevOps {self.project_root_path}")
self.module = module
if not name:
self.name = module
self.build_dir_name = build_dir_name
# Deprecated - no longer use generic stack ...
self.stack = {}
@deprecation.deprecated(deprecated_in="3.2")
# use .name instead
def name(self):
return self.name
def build_path(self):
path = [self.project_root_path, self.build_dir_name, self.name, self.module]
logging.warn(f"Set project build_path in Devops {path}")
return "/".join(filter_none(path))
def __put__(self, key, value):
self.stack[key] = value
def __get__(self, key):
return self.stack[key]
def __get_keys__(self, keys):
result = {}
for key in keys:
result[key] = self.__get__(key)
return result

View file

@ -0,0 +1,30 @@
from typing import Optional
from .common import (
filter_none,
Validateable,
Devops,
)
class Image(Validateable):
def __init__(
self,
dockerhub_user,
dockerhub_password,
devops: Devops,
build_dir_name="target",
use_package_common_files=True,
build_commons_path=None,
docker_build_commons_dir_name="docker",
docker_publish_tag=None,
):
self.dockerhub_user = dockerhub_user
self.dockerhub_password = dockerhub_password
self.use_package_common_files = use_package_common_files
self.build_commons_path = build_commons_path
self.docker_build_commons_dir_name = docker_build_commons_dir_name
self.docker_publish_tag = docker_publish_tag
self.devops = devops
def docker_build_commons_path(self):
list = [self.build_commons_path, self.docker_build_commons_dir_name]
return "/".join(filter_none(list)) + "/"

View file

@ -0,0 +1,122 @@
from enum import Enum
from typing import Optional
from pathlib import Path
from .common import (
filter_none,
Validateable,
Devops,
)
class ReleaseType(Enum):
MAJOR = 0
MINOR = 1
PATCH = 2
SNAPSHOT = 3
BUMP = None
class EnvironmentKeys(Enum):
DDADEVOPS_RELEASE_TYPE = 0
class Version():
def __init__(self, id: Path, version_list: list):
self.id = id
self.version_list = version_list
self.version_string: Optional[str | None] = None
self.is_snapshot: Optional[bool | None] = None
def increment(self, release_type: ReleaseType):
self.is_snapshot = False
match release_type:
case ReleaseType.BUMP:
self.is_snapshot = True
self.version_list[ReleaseType.PATCH.value] += 1
case ReleaseType.SNAPSHOT:
self.is_snapshot = True
case ReleaseType.PATCH:
self.version_list[ReleaseType.PATCH.value] += 1
case ReleaseType.MINOR:
self.version_list[ReleaseType.PATCH.value] = 0
self.version_list[ReleaseType.MINOR.value] += 1
case ReleaseType.MAJOR:
self.version_list[ReleaseType.PATCH.value] = 0
self.version_list[ReleaseType.MINOR.value] = 0
self.version_list[ReleaseType.MAJOR.value] += 1
case None:
raise Exception("Release Type was not set!")
def get_version_string(self) -> str:
self.version_string = ".".join([str(x) for x in self.version_list])
if self.is_snapshot:
self.version_string += "-SNAPSHOT"
return self.version_string
def create_release_version(self, release_type: ReleaseType):
release_version = Version(self.id, self.version_list.copy())
release_version.is_snapshot = self.is_snapshot
release_version.increment(release_type)
return release_version
def create_bump_version(self):
bump_version = Version(self.id, self.version_list.copy())
bump_version.is_snapshot = self.is_snapshot
bump_version.increment(ReleaseType.BUMP)
return bump_version
class ReleaseContext(Validateable):
def __init__(self, release_type: ReleaseType | None, version: Version, current_branch: str):
self.release_type = release_type
self.version = version
self.current_branch = current_branch
def release_version(self) -> Version:
return self.version.create_release_version(self.release_type)
def bump_version(self) -> Version:
return self.release_version().create_bump_version()
def validate(self):
result = []
result += self.__validate_is_not_empty__("release_type")
result += self.__validate_is_not_empty__("version")
result += self.__validate_is_not_empty__("current_branch")
return result
def validate_branch(self, main_branch: str):
result = []
if self.release_type is not None and main_branch != self.current_branch:
result.append(f"Releases are allowed only on {main_branch}")
return result
class Release(Validateable):
def __init__(
self,
devops: Devops,
main_branch: str,
config_file: str,
):
self.devops = devops
self.main_branch = main_branch
self.config_file = config_file
self.release_context = None
def set_release_context(self, set_release_context: ReleaseContext) -> None:
self.release_context = set_release_context
def release_version(self):
return self.release_context.release_version()
def bump_version(self):
return self.release_context.bump_version()
def validate(self):
result = []
result += self.__validate_is_not_empty__("main_branch")
result += self.__validate_is_not_empty__("config_file")
result += self.__validate_is_not_empty__("release_context")
if self.release_context is not None:
result += self.release_context.validate()
result += self.release_context.validate_branch(self.main_branch)
return result

View file

@ -0,0 +1 @@
from .infrastructure import FileApi, ImageApi, ResourceApi, ExecutionApi, ProjectRepository

View file

@ -0,0 +1,128 @@
from pathlib import Path
from sys import stdout
from os import chmod
from subprocess import run
from pkg_resources import resource_string
import yaml
from ..domain import Devops, Image, C4k, Release
from ..python_util import execute
class ProjectRepository:
def set_build(self, project, build):
project.set_property("devops_build", build)
def get_devops(self, project) -> Devops:
return project.get_property("build")
def set_devops(self, project, build: Devops):
project.set_property("build", build)
def get_docker(self, project) -> Image:
return project.get_property("docker_build")
def set_docker(self, project, build: Image):
project.set_property("docker_build", build)
def get_c4k(self, project) -> C4k:
return project.get_property("c4k_build")
def set_c4k(self, project, build: C4k):
project.set_property("c4k_build", build)
def get_release(self, project) -> Release:
return project.get_property("release_build")
def set_release(self, project, build: Release):
project.set_property("release_build", build)
class ResourceApi:
def read_resource(self, path: str) -> bytes:
return resource_string(__name__, path)
class FileApi:
def clean_dir(self, directory: str):
execute("rm -rf " + directory, shell=True)
execute("mkdir -p " + directory, shell=True)
def cp_force(self, src: str, target_dir: str):
execute("cp -f " + src + "* " + target_dir, shell=True)
def cp_recursive(self, src: str, target_dir: str):
execute("cp -r " + src + " " + target_dir, shell=True)
def write_data_to_file(self, path: Path, data: bytes):
with open(path, "w", encoding="utf-8") as output_file:
output_file.write(data.decode(stdout.encoding))
def write_yaml_to_file(self, path: Path, data: map):
with open(path, "w", encoding="utf-8") as output_file:
yaml.dump(data, output_file)
chmod(path, 0o600)
class ImageApi:
def image(self, name: str, path: Path):
run(
f"docker build -t {name} --file {path}/image/Dockerfile {path}/image",
shell=True,
check=True,
)
def drun(self, name: str):
run(
f"docker run -it --entrypoint=\"\" {name} /bin/bash",
shell=True,
check=True,
)
def dockerhub_login(self, username: str, password: str):
run(
f"docker login --username {username} --password {password}",
shell=True,
check=True,
)
def dockerhub_publish(self, name: str, username: str, tag=None):
if tag is not None:
run(
f"docker tag {name} {username}/{name}:{tag}",
shell=True,
check=True,
)
run(
f"docker push {username}/{name}:{tag}",
shell=True,
check=True,
)
run(
f"docker tag {name} {username}/{name}:latest",
shell=True,
check=True,
)
run(
f"docker push {username}/{name}:latest",
shell=True,
check=True,
)
def test(self, name: str, path: Path):
run(
f"docker build -t {name} -test --file {path}/test/Dockerfile {path}/test",
shell=True,
check=True,
)
class ExecutionApi:
def execute(self, command: str, dry_run=False):
output = ""
if dry_run:
print(command)
else:
output = execute(command, True)
print(output)
return output

View file

@ -0,0 +1,2 @@
from .infrastructure_api import FileHandler, EnvironmentApi, GitApi, JsonFileHandler, GradleFileHandler, PythonFileHandler, ClojureFileHandler
from .repo import VersionRepository, ReleaseContextRepository, ReleaseTypeRepository

View file

@ -0,0 +1,243 @@
import json
import re
import subprocess as sub
from abc import ABC, abstractmethod
from typing import Optional
from pathlib import Path
from os import environ
from ..infrastructure import ExecutionApi
# TODO: jem, zam - 2023_04_18: Discuss if we can move more functionality to domain?
class FileHandler(ABC):
def __init__(self) -> None:
self.config_file_path: Optional[Path | None] = None
self.config_file_type: Optional[Path | None] = None
@classmethod
def from_file_path(cls, file_path):
config_file_type = file_path.suffix
match config_file_type:
case '.json':
file_handler = JsonFileHandler()
case '.gradle':
file_handler = GradleFileHandler()
case '.clj':
file_handler = ClojureFileHandler()
case '.py':
file_handler = PythonFileHandler()
case _:
raise Exception(
f'The file type "{config_file_type}" is not implemented')
# TODO: Attribute is only set in classmethod. Should this be initialized outside of this class?
file_handler.config_file_path = file_path
file_handler.config_file_type = config_file_type
return file_handler
@abstractmethod
def parse(self) -> tuple[list[int], bool]:
pass
@abstractmethod
def write(self, version_string):
pass
class JsonFileHandler(FileHandler):
def parse(self) -> tuple[list[int], bool]:
if self.config_file_path is None:
raise ValueError("No file name given.")
with open(self.config_file_path, 'r') as json_file:
json_version = json.load(json_file)['version']
is_snapshot = False
if '-SNAPSHOT' in json_version:
is_snapshot = True
json_version = json_version.replace('-SNAPSHOT', '')
version = [int(x) for x in json_version.split('.')]
return version, is_snapshot
def write(self, version_string):
with open(self.config_file_path, 'r+') as json_file:
json_data = json.load(json_file)
json_data['version'] = version_string
json_file.seek(0)
json.dump(json_data, json_file, indent=4)
json_file.truncate()
class GradleFileHandler(FileHandler):
def parse(self) -> tuple[list[int], bool]:
if self.config_file_path is None:
raise ValueError("No file name given.")
with open(self.config_file_path, 'r') as gradle_file:
contents = gradle_file.read()
version_line = re.search("\nversion = .*", contents)
exception = Exception("Version not found in gradle file")
if version_line is None:
raise exception
version_line_group = version_line.group()
version_string = re.search(
'[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group)
if version_string is None:
raise exception
version_string_group = version_string.group()
is_snapshot = False
if '-SNAPSHOT' in version_string_group:
is_snapshot = True
version_string_group = version_string_group.replace('-SNAPSHOT', '')
version = [int(x) for x in version_string_group.split('.')]
return version, is_snapshot
def write(self, version_string):
with open(self.config_file_path, 'r+') as gradle_file:
contents = gradle_file.read()
version_substitute = re.sub(
'\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents)
gradle_file.seek(0)
gradle_file.write(version_substitute)
gradle_file.truncate()
class PythonFileHandler(FileHandler):
def parse(self) -> tuple[list[int], bool]:
if self.config_file_path is None:
raise ValueError("No file name given.")
with open(self.config_file_path, 'r') as python_file:
contents = python_file.read()
version_line = re.search("\nversion = .*\n", contents)
exception = Exception("Version not found in gradle file")
if version_line is None:
raise exception
version_line_group = version_line.group()
version_string = re.search(
'[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group)
if version_string is None:
raise exception
version_string_group = version_string.group()
is_snapshot = False
if '-SNAPSHOT' in version_string_group:
is_snapshot = True
version_string_group = version_string_group.replace('-SNAPSHOT', '')
version = [int(x) for x in version_string_group.split('.')]
return version, is_snapshot
def write(self, version_string):
with open(self.config_file_path, 'r+') as python_file:
contents = python_file.read()
version_substitute = re.sub(
'\nversion = "[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?"', f'\nversion = "{version_string}"', contents)
python_file.seek(0)
python_file.write(version_substitute)
python_file.truncate()
class ClojureFileHandler(FileHandler):
def parse(self) -> tuple[list[int], bool]:
if self.config_file_path is None:
raise ValueError("No file name given.")
with open(self.config_file_path, 'r') as clj_file:
contents = clj_file.read()
version_line = re.search("^\\(defproject .*\n", contents)
exception = Exception("Version not found in clj file")
if version_line is None:
raise exception
version_line_group = version_line.group()
version_string = re.search(
'[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', version_line_group)
if version_string is None:
raise exception
version_string_group = version_string.group()
is_snapshot = False
if '-SNAPSHOT' in version_string_group:
is_snapshot = True
version_string_group = version_string_group.replace('-SNAPSHOT', '')
version = [int(x) for x in version_string_group.split('.')]
return version, is_snapshot
def write(self, version_string):
with open(self.config_file_path, 'r+') as clj_file:
clj_first = clj_file.readline()
clj_rest = clj_file.read()
version_substitute = re.sub(
'[0-9]*\\.[0-9]*\\.[0-9]*(-SNAPSHOT)?', f'"{version_string}"\n', clj_first)
clj_file.seek(0)
clj_file.write(version_substitute)
clj_file.write(clj_rest)
clj_file.truncate()
class GitApi():
def __init__(self):
self.execution_api = ExecutionApi()
def get_latest_n_commits(self, n: int):
return self.execution_api.execute(
f'git log --oneline --format="%s %b" -n {n}')
def get_latest_commit(self):
return self.get_latest_n_commits(1)
def tag_annotated(self, annotation: str, message: str, count: int):
return self.execution_api.execute(
f'git tag -a {annotation} -m {message} HEAD~{count}')
def tag_annotated_second_last(self, annotation: str, message:str):
return self.tag_annotated(annotation, message, 1)
def get_latest_tag(self):
return self.execution_api.execute('git describe --tags --abbrev=0')
def get_current_branch(self):
self.execution_api.execute('git branch --show-current')
return ''.join(self.execution_api.stdout).rstrip()
def init(self, default_branch: str = "main"):
self.execution_api.execute(f'git init')
self.execution_api.execute(f'git checkout -b {default_branch}')
def set_user_config(self, email: str, name: str):
self.execution_api.execute(f'git config user.email {email}')
self.execution_api.execute(f'git config user.name {name}')
def add_file(self, file_path: Path):
return self.execution_api.execute(f'git add {file_path}')
def add_remote(self, origin: str, url: str):
return self.execution_api.execute(f'git remote add {origin} {url}')
def commit(self, commit_message: str):
return self.execution_api.execute(
f'git commit -m "{commit_message}"')
def push(self):
return self.execution_api.execute('git push')
def checkout(self, branch: str):
return self.execution_api.execute(f'git checkout {branch}')
class EnvironmentApi():
def __init__(self):
self.environ = environ
def get(self, key):
return self.environ.get(key)
def set(self, key, value):
self.environ[key] = value

View file

@ -0,0 +1,130 @@
from typing import Optional
from src.main.python.ddadevops.domain import (
ReleaseContext,
Version,
ReleaseType,
EnvironmentKeys,
)
from src.main.python.ddadevops.infrastructure.release_mixin import (
FileHandler,
GitApi,
EnvironmentApi,
)
class VersionRepository:
def __init__(self, file):
self.file = file
self.file_handler = None
def load_file(self):
self.file_handler = FileHandler.from_file_path(self.file)
return self.file_handler
def write_file(self, version_string):
if self.file_handler is None:
raise Exception("Version was not created by load_file method.")
else:
self.file_handler.write(version_string)
def parse_file(self):
version_list, is_snapshot = self.file_handler.parse()
return version_list, is_snapshot
def get_version(self) -> Version:
self.file_handler = self.load_file()
version_list, is_snapshot = self.parse_file()
version = Version(self.file, version_list)
version.is_snapshot = is_snapshot
return version
class ReleaseTypeRepository:
def __init__(
self,
git_api: GitApi = GitApi(),
environment_api: EnvironmentApi = EnvironmentApi(),
):
self.git_api: GitApi = git_api
self.environment_api: EnvironmentApi = environment_api
self.get_from_git: bool = False
self.get_from_env: bool = False
@classmethod
def from_git(cls, git_api: GitApi):
releaseTypeRepo = cls(git_api=git_api)
releaseTypeRepo.get_from_git = True
return releaseTypeRepo
@classmethod
def from_environment(cls, environment_api: EnvironmentApi):
releaseTypeRepo = cls(environment_api=environment_api)
releaseTypeRepo.get_from_env = True
return releaseTypeRepo
def __get_release_type_git(self) -> ReleaseType | None:
latest_commit = self.git_api.get_latest_commit()
if ReleaseType.MAJOR.name in latest_commit.upper():
return ReleaseType.MAJOR
elif ReleaseType.MINOR.name in latest_commit.upper():
return ReleaseType.MINOR
elif ReleaseType.PATCH.name in latest_commit.upper():
return ReleaseType.PATCH
elif ReleaseType.SNAPSHOT.name in latest_commit.upper():
return ReleaseType.SNAPSHOT
else:
return None
def __get_release_type_environment(self) -> ReleaseType | None:
release_name = self.environment_api.get(
EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name
)
if release_name is None:
raise ValueError(
"Release Name not found. Is the Environment correctly configured?"
)
elif ReleaseType.MAJOR.name in release_name.upper():
return ReleaseType.MAJOR
elif ReleaseType.MINOR.name in release_name.upper():
return ReleaseType.MINOR
elif ReleaseType.PATCH.name in release_name.upper():
return ReleaseType.PATCH
elif ReleaseType.SNAPSHOT.name in release_name.upper():
return ReleaseType.SNAPSHOT
else:
return None
def get_release_type(self) -> ReleaseType | None:
if self.get_from_git:
return self.__get_release_type_git()
elif self.get_from_env:
return self.__get_release_type_environment()
else:
raise ValueError("No valid api passed to ReleaseTypeRepository")
# TODO: Repo has state & repository should exist only for AggregateRoot
class ReleaseContextRepository:
def __init__(
self,
version_repository: VersionRepository,
release_type_repository: ReleaseTypeRepository,
main_branch: str,
):
self.version_repository = version_repository
self.release_type_repository = release_type_repository
self.main_branch = main_branch
def get_release(self) -> ReleaseContext:
result = ReleaseContext(
self.release_type_repository.get_release_type(),
self.version_repository.get_version(),
self.main_branch,
)
if not result.is_valid():
issues = '\n'.join(result.validate())
raise ValueError(f"invalid release: {issues}")
return result

View file

@ -1,11 +1,7 @@
from subprocess import check_output, Popen, PIPE from subprocess import check_output, Popen, PIPE
import sys
def execute(cmd, shell=False): def execute(cmd, shell=False):
if sys.version_info.major == 3: output = check_output(cmd, encoding='UTF-8', shell=shell)
output = check_output(cmd, encoding='UTF-8', shell=shell)
else:
output = check_output(cmd, shell=shell)
return output.rstrip() return output.rstrip()
def execute_live(cmd): def execute_live(cmd):

View file

@ -0,0 +1,36 @@
from typing import Optional
from pybuilder.core import Project
from src.main.python.ddadevops.devops_build import DevopsBuild
from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseContextRepository, ReleaseTypeRepository, VersionRepository, GitApi, EnvironmentApi
from src.main.python.ddadevops.application import PrepareReleaseService, TagAndPushReleaseService
from src.main.python.ddadevops.domain import Release, EnvironmentKeys
class ReleaseMixin(DevopsBuild):
def __init__(self, project: Project, release: Release):
super().__init__(project, devops=release.devops)
self.repo.set_release(self.project, release)
git_api = GitApi()
environment_api = EnvironmentApi()
env_key = EnvironmentKeys.DDADEVOPS_RELEASE_TYPE.name
environment_val_set = environment_api.get(env_key) != "" and environment_api.get(env_key) is not None
if environment_val_set:
release_type_repo = ReleaseTypeRepository.from_environment(environment_api)
else:
release_type_repo = ReleaseTypeRepository.from_git(git_api)
version_repo = VersionRepository(release.config_file)
self.release_repo = ReleaseContextRepository(version_repo, release_type_repo, release.main_branch)
self.prepare_release_service = PrepareReleaseService()
self.tag_and_push_release_service = TagAndPushReleaseService(git_api)
def prepare_release(self):
release = self.release_repo.get_release()
self.prepare_release_service.write_and_commit_release(release, self.release_repo.version_repository)
self.prepare_release_service.write_and_commit_bump(release, self.release_repo.version_repository)
def tag_and_push_release(self):
self.tag_and_push_release_service.tag_release(self.release_repo)
self.tag_and_push_release_service.push_release()

View file

View file

View file

@ -0,0 +1,226 @@
from pybuilder.core import Project
from pathlib import Path
from src.main.python.ddadevops.domain.common import (
Validateable,
DnsRecord,
Devops,
)
from src.main.python.ddadevops.domain import Version, ReleaseType, Release, ReleaseContext
from src.main.python.ddadevops.domain.image import Image
from src.main.python.ddadevops.domain.c4k import C4k
from src.main.python.ddadevops.c4k_mixin import add_c4k_mixin_config
class MockValidateable(Validateable):
def __init__(self, value):
self.field = value
def validate(self):
return self.__validate_is_not_empty__("field")
def test_should_validate_non_empty_strings():
sut = MockValidateable("content")
assert sut.is_valid()
sut = MockValidateable(None)
assert not sut.is_valid()
sut = MockValidateable("")
assert not sut.is_valid()
def test_should_validate_non_empty_others():
sut = MockValidateable(1)
assert sut.is_valid()
sut = MockValidateable(1.0)
assert sut.is_valid()
sut = MockValidateable(True)
assert sut.is_valid()
sut = MockValidateable(None)
assert not sut.is_valid()
def test_validate_with_reason():
sut = MockValidateable(None)
assert sut.validate()[0] == "Field 'field' must not be empty."
def test_should_validate_DnsRecord():
sut = DnsRecord(None)
assert not sut.is_valid()
sut = DnsRecord("name")
assert not sut.is_valid()
sut = DnsRecord("name", ipv4="1.2.3.4")
assert sut.is_valid()
sut = DnsRecord("name", ipv6="1::")
assert sut.is_valid()
def test_devops_buildpath():
sut = Devops(
stage="test", project_root_path="../../..", module="cloud", name="meissa"
)
assert "../../../target/meissa/cloud" == sut.build_path()
def test_devops_build_commons_path():
devops = Devops(
stage="test", project_root_path="../../..", module="cloud", name="meissa"
)
sut = Image(
dockerhub_user="user",
dockerhub_password="password",
devops = devops,
)
assert "docker/" == sut.docker_build_commons_path()
def test_c4k_build_should_update_fqdn():
project_config = {
"stage": "test",
"name": "name",
"project_root_path": "mypath",
"module": "module",
"build_dir_name": "target",
}
config = {"issuer": "staging"}
auth = {
"jvb-auth-password": "pw1",
"jicofo-auth-password": "pw2",
"jicofo-component-secret": "pw3",
}
add_c4k_mixin_config(
project_config,
config,
auth,
grafana_cloud_user="user",
grafana_cloud_password="password",
)
sut = C4k(project_config)
sut.update_runtime_config(DnsRecord("test.de", ipv6="1::"))
assert {
"issuer": "staging",
"fqdn": "test.de",
"mon-cfg": {
"cluster-name": "module",
"cluster-stage": "test",
"grafana-cloud-url": "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push",
},
} == sut.config()
assert {
"jicofo-auth-password": "pw2",
"jicofo-component-secret": "pw3",
"jvb-auth-password": "pw1",
"mon-auth": {
"grafana-cloud-password": "password",
"grafana-cloud-user": "user",
},
} == sut.c4k_mixin_auth
def test_c4k_build_should_calculate_command():
devops = Devops(stage="test", project_root_path="", module="module", name="name")
project_config = {
"stage": "test",
"name": "name",
"project_root_path": "",
"module": "module",
"build_dir_name": "target",
}
add_c4k_mixin_config(
project_config,
{},
{},
grafana_cloud_user="user",
grafana_cloud_password="password",
)
sut = C4k(project_config)
assert (
"c4k-module-standalone.jar "
+ "/target/name/module/out_c4k_config.yaml "
+ "/target/name/module/out_c4k_auth.yaml > "
+ "/target/name/module/out_module.yaml"
== sut.command(devops)
)
project_config = {
"stage": "test",
"name": "name",
"project_root_path": "",
"module": "module",
"build_dir_name": "target",
}
add_c4k_mixin_config(
project_config,
{},
{},
executabel_name="executabel_name",
grafana_cloud_user="user",
grafana_cloud_password="password",
)
sut = C4k(project_config)
assert (
"c4k-executabel_name-standalone.jar "
+ "/target/name/module/out_c4k_config.yaml "
+ "/target/name/module/out_c4k_auth.yaml > "
+ "/target/name/module/out_module.yaml"
== sut.command(devops)
)
def test_version(tmp_path: Path):
version = Version(tmp_path, [1, 2, 3])
version.increment(ReleaseType.SNAPSHOT)
assert version.get_version_string() == "1.2.3-SNAPSHOT"
assert version.version_list == [1, 2, 3]
assert version.is_snapshot
version = Version(tmp_path, [1, 2, 3])
version.increment(ReleaseType.BUMP)
assert version.get_version_string() == "1.2.4-SNAPSHOT"
assert version.version_list == [1, 2, 4]
assert version.is_snapshot
version = Version(tmp_path, [1, 2, 3])
version.increment(ReleaseType.PATCH)
assert version.get_version_string() == "1.2.4"
assert version.version_list == [1, 2, 4]
assert not version.is_snapshot
version = Version(tmp_path, [1, 2, 3])
version.increment(ReleaseType.MINOR)
assert version.get_version_string() == "1.3.0"
assert version.version_list == [1, 3, 0]
assert not version.is_snapshot
version = Version(tmp_path, [1, 2, 3])
version.increment(ReleaseType.MAJOR)
assert version.get_version_string() == "2.0.0"
assert version.version_list == [2, 0, 0]
assert not version.is_snapshot
def test_release_context(tmp_path):
version = Version(tmp_path, [1, 2, 3])
release = ReleaseContext(ReleaseType.MINOR, version, "main")
release_version = release.release_version()
assert release_version.get_version_string() in '1.3.0'
bump_version = release.bump_version()
assert bump_version.get_version_string() in "1.3.1-SNAPSHOT"
def test_release(tmp_path):
devops = Devops(stage="test", project_root_path="", module="module", name="name")
sut = Release(devops, "main", "config_file.json")
assert not sut.is_valid()
sut.set_release_context(ReleaseContext(ReleaseType.MINOR, Version("id", [1,2,3]), "main"))
assert sut.is_valid()

View file

@ -0,0 +1,2 @@
from .mock_infrastructure import MockReleaseRepository, MockReleaseTypeRepository, MockVersionRepository
from .mock_infrastructure_api import MockGitApi

View file

@ -0,0 +1,12 @@
from pathlib import Path
from src.main.python.ddadevops.infrastructure import ExecutionApi
class Helper():
def __init__(self, file_name = 'config.json'):
self.TEST_FILE_NAME = file_name
self.TEST_FILE_ROOT = Path('src/test/resources/')
self.TEST_FILE_PATH = self.TEST_FILE_ROOT / self.TEST_FILE_NAME
def copy_files(self, source: Path, target: Path):
api = ExecutionApi()
api.execute(f"cp {source} {target}")

View file

@ -0,0 +1,43 @@
from pathlib import Path
from src.main.python.ddadevops.domain import ReleaseType, Version, ReleaseContext
from .mock_infrastructure_api import MockGitApi
class MockVersionRepository():
def __init__(self):
self.file = None
self.file_handler = None
self.write_file_count = 0
def load_file(self):
pass
def write_file(self, version_string):
self.write_file_count += 1
pass
def parse_file(self):
pass
def get_version(self) -> Version:
return Version(Path(), [0,0,0])
class MockReleaseTypeRepository():
def __init__(self, mock_git_api: MockGitApi):
self.git_api = mock_git_api
def get_release_type(self):
return ReleaseType.MINOR
class MockReleaseRepository():
def __init__(self, version_repository: MockVersionRepository, release_type_repository: MockReleaseTypeRepository, main_branch: str):
self.version_repository = version_repository
self.release_type_repository = release_type_repository
self.main_branch = main_branch
self.get_release_count = 0
def get_release(self) -> ReleaseContext:
self.get_release_count += 1
return ReleaseContext(self.release_type_repository.get_release_type(), self.version_repository.get_version(), self.main_branch)

View file

@ -0,0 +1,73 @@
class MockSystemApi():
def __init__(self):
self.stdout = [""]
self.stderr = [""]
def run(self, args):
pass
def run_checked(self, *args):
self.run(args)
pass
class MockGitApi():
def __init__(self, commit_string = ""):
self.system_api = MockSystemApi()
self.get_latest_commit_count = 0
self.commit_string = commit_string
self.tag_annotated_count = 0
self.add_file_count = 0
self.commit_count = 0
self.push_count = 0
def get_latest_n_commits(self, n: int):
return " "
def get_latest_commit(self):
self.get_latest_commit_count += 1
return self.commit_string
def tag_annotated(self, annotation: str, message: str, count: int):
self.tag_annotated_count += 1
return " "
def tag_annotated_second_last(self, annotation: str, message: str):
self.tag_annotated(annotation, message, 1)
return " "
def get_latest_tag(self):
return " "
def get_current_branch(self):
return " "
def init(self):
pass
def add_file(self, file_path):
self.add_file_count += 1
return " "
def commit(self, commit_message: str):
self.commit_count += 1
return commit_message
def push(self):
self.push_count += 1
return " "
def checkout(self, branch: str):
return " "
class MockEnvironmentApi():
def __init__(self, environ_map):
self.environ = environ_map
def get(self, name):
return self.environ.get(name)
def set(self, name, value):
self.environ[name] = value

View file

@ -0,0 +1,86 @@
from src.main.python.ddadevops.domain import ReleaseType
from src.main.python.ddadevops.infrastructure.release_mixin import ReleaseTypeRepository, VersionRepository, ReleaseContextRepository
from .mock_infrastructure_api import MockGitApi, MockEnvironmentApi
from .helper import Helper
def test_version_repository(tmp_path):
# init
th = Helper()
th.copy_files(th.TEST_FILE_PATH, tmp_path)
sut = VersionRepository(th.TEST_FILE_PATH)
version = sut.get_version()
#test
assert version is not None
def test_release_repository(tmp_path):
# init
th = Helper()
th.copy_files( th.TEST_FILE_PATH, tmp_path)
version_repo = VersionRepository(th.TEST_FILE_PATH)
release_type_repo = ReleaseTypeRepository.from_git(MockGitApi('MINOR test'))
# test
sut = ReleaseContextRepository(version_repo, release_type_repo, 'main')
release = sut.get_release()
assert release is not None
def test_release_type_repository_git():
sut = ReleaseTypeRepository.from_git(MockGitApi('MINOR test'))
release_type = sut.get_release_type()
assert release_type is ReleaseType.MINOR
sut = ReleaseTypeRepository.from_git(MockGitApi('MINOR bla'))
release_type = sut.get_release_type()
assert release_type is ReleaseType.MINOR
sut = ReleaseTypeRepository.from_git(MockGitApi('Major bla'))
release_type = sut.get_release_type()
assert release_type == ReleaseType.MAJOR
sut = ReleaseTypeRepository.from_git(MockGitApi('PATCH bla'))
release_type = sut.get_release_type()
assert release_type == ReleaseType.PATCH
sut = ReleaseTypeRepository.from_git(MockGitApi('SNAPSHOT bla'))
release_type = sut.get_release_type()
assert release_type == ReleaseType.SNAPSHOT
sut = ReleaseTypeRepository.from_git(MockGitApi('bla'))
release_type = sut.get_release_type()
assert release_type == None
def test_release_type_repository_env():
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'MINOR test'}))
release_type = sut.get_release_type()
assert release_type is ReleaseType.MINOR
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'MINOR'}))
release_type = sut.get_release_type()
assert release_type is ReleaseType.MINOR
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Major bla'}))
release_type = sut.get_release_type()
assert release_type == ReleaseType.MAJOR
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Patch bla'}))
release_type = sut.get_release_type()
assert release_type == ReleaseType.PATCH
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Snapshot bla'}))
release_type = sut.get_release_type()
assert release_type == ReleaseType.SNAPSHOT
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'DDADEVOPS_RELEASE_TYPE': 'Random text'}))
release_type = sut.get_release_type()
assert release_type == None
sut = ReleaseTypeRepository.from_environment(MockEnvironmentApi({'REL_TYPE': 'Not the right variable'}))
try:
release_type = sut.get_release_type()
except:
assert release_type == None

View file

@ -0,0 +1,103 @@
from pathlib import Path
import pytest as pt
from src.main.python.ddadevops.infrastructure.release_mixin import GitApi, EnvironmentApi, JsonFileHandler
from src.main.python.ddadevops.infrastructure.release_mixin import VersionRepository
from src.main.python.ddadevops.domain.release import ReleaseType
from .helper import Helper
def change_test_dir( tmp_path: Path, monkeypatch: pt.MonkeyPatch):
monkeypatch.chdir(tmp_path)
def test_environment_api():
# init
env_api = EnvironmentApi()
key = "TEST_ENV_KEY"
value = "data"
env_api.set(key, value)
#test
assert env_api.get(key) == value
def test_git_api(tmp_path: Path, monkeypatch: pt.MonkeyPatch):
# init
th = Helper()
th.copy_files(th.TEST_FILE_PATH, tmp_path)
# change the context of the script execution to tmp_path
change_test_dir(tmp_path, monkeypatch)
git_api = GitApi()
git_api.init()
git_api.set_user_config("ex.ample@mail.com", "Ex Ample")
git_api.add_file(th.TEST_FILE_NAME)
git_api.commit("MINOR release")
# test
latest_commit = git_api.get_latest_commit()
assert "MINOR release" in latest_commit
# file handler tests
def test_gradle(tmp_path):
# init
th = Helper('config.gradle')
th.copy_files(th.TEST_FILE_PATH, tmp_path)
th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME
# test
repo = VersionRepository(th.TEST_FILE_PATH)
version = repo.get_version()
version = version.create_release_version(ReleaseType.SNAPSHOT)
repo.write_file(version.get_version_string())
# check
assert 'version = "12.4.678-SNAPSHOT"' in th.TEST_FILE_PATH.read_text()
def test_json(tmp_path):
# init
th = Helper('config.json')
th.copy_files(th.TEST_FILE_PATH, tmp_path)
th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME
# test
repo = VersionRepository(th.TEST_FILE_PATH)
version = repo.get_version()
version = version.create_release_version(ReleaseType.SNAPSHOT)
repo.write_file(version.get_version_string())
# check
assert '"version": "123.123.456-SNAPSHOT"' in th.TEST_FILE_PATH.read_text()
def test_clojure(tmp_path):
# init
th = Helper('config.clj')
th.copy_files(th.TEST_FILE_PATH, tmp_path)
th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME
# test
repo = VersionRepository(th.TEST_FILE_PATH)
version = repo.get_version()
version = version.create_release_version(ReleaseType.SNAPSHOT)
repo.write_file(version.get_version_string())
# check
assert '1.1.3-SNAPSHOT' in th.TEST_FILE_PATH.read_text()
def test_python(tmp_path):
# init
th = Helper('config.py')
th.copy_files(th.TEST_FILE_PATH, tmp_path)
th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME
# test
repo = VersionRepository(th.TEST_FILE_PATH)
version = repo.get_version()
version = version.create_release_version(ReleaseType.SNAPSHOT)
repo.write_file(version.get_version_string())
# check
assert '3.1.3-SNAPSHOT' in th.TEST_FILE_PATH.read_text()

View file

@ -0,0 +1,80 @@
import pytest as pt
from pathlib import Path
from pybuilder.core import Project
from src.main.python.ddadevops.release_mixin import ReleaseMixin
from src.main.python.ddadevops.infrastructure.release_mixin import GitApi, EnvironmentApi
from src.main.python.ddadevops.domain import Devops, ReleaseContext, Release
from .helper import Helper
MAIN_BRANCH = 'main'
STAGE = 'test'
PROJECT_ROOT_PATH = '.'
MODULE = 'test'
BUILD_DIR_NAME = "build_dir"
def change_test_dir( tmp_path: Path, monkeypatch: pt.MonkeyPatch):
monkeypatch.chdir(tmp_path)
class MyBuild(ReleaseMixin):
pass
def initialize_with_object(project, CONFIG_FILE):
project.build_depends_on('ddadevops>=3.1.2')
devops = Devops(STAGE, PROJECT_ROOT_PATH, MODULE, "release_test", BUILD_DIR_NAME)
release = Release(devops, MAIN_BRANCH, CONFIG_FILE)
build = MyBuild(project, release)
return build
def test_release_mixin_git(tmp_path: Path, monkeypatch: pt.MonkeyPatch):
# init
th = Helper()
th.copy_files(th.TEST_FILE_PATH, tmp_path)
th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME
change_test_dir(tmp_path, monkeypatch)
project = Project(tmp_path)
git_api = GitApi()
git_api.init()
git_api.set_user_config("ex.ample@mail.com", "Ex Ample")
git_api.add_file(th.TEST_FILE_NAME)
git_api.commit("MAJOR release")
build = initialize_with_object(project, th.TEST_FILE_PATH)
build.prepare_release()
release_version = build.release_repo.version_repository.get_version()
# test
assert "124.0.1-SNAPSHOT" in release_version.get_version_string()
def test_release_mixin_environment(tmp_path: Path, monkeypatch: pt.MonkeyPatch):
# init
th = Helper()
th.copy_files(th.TEST_FILE_PATH, tmp_path)
th.TEST_FILE_PATH = tmp_path / th.TEST_FILE_NAME
change_test_dir(tmp_path, monkeypatch)
project = Project(tmp_path)
git_api = GitApi()
git_api.init()
git_api.set_user_config("ex.ample@mail.com", "Ex Ample")
git_api.add_file(th.TEST_FILE_NAME)
git_api.commit("Commit Message")
environment_api = EnvironmentApi()
environment_api.set("DDADEVOPS_RELEASE_TYPE", "MAJOR")
build = initialize_with_object(project, th.TEST_FILE_PATH)
build.prepare_release()
release_version = build.release_repo.version_repository.get_version()
# test
assert "124.0.1-SNAPSHOT" in release_version.get_version_string()
# tear down
environment_api.set("DDADEVOPS_RELEASE_TYPE", "")

View file

@ -0,0 +1,32 @@
from src.main.python.ddadevops.application import PrepareReleaseService, TagAndPushReleaseService
from src.test.python.release_mixin import MockReleaseRepository, MockReleaseTypeRepository, MockVersionRepository
from src.test.python.release_mixin import MockGitApi
def test_prepare_release_service():
# init
mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi()), 'main')
prepare_release_service = PrepareReleaseService()
prepare_release_service.git_api = MockGitApi()
prepare_release_service.write_and_commit_release(mock_release_repo.get_release(), mock_release_repo.version_repository)
#test
assert prepare_release_service.git_api.add_file_count == 1
assert prepare_release_service.git_api.commit_count == 1
# init
prepare_release_service.write_and_commit_bump(mock_release_repo.get_release(), mock_release_repo.version_repository)
# test
assert prepare_release_service.git_api.add_file_count == 2
assert prepare_release_service.git_api.commit_count == 2
def test_tag_and_push_release_service():
# init
mock_release_repo = MockReleaseRepository(MockVersionRepository(), MockReleaseTypeRepository(MockGitApi()), 'main')
tag_and_push_release_service = TagAndPushReleaseService(MockGitApi())
tag_and_push_release_service.tag_release(mock_release_repo)
tag_and_push_release_service.push_release()
#test
assert tag_and_push_release_service.git_api.tag_annotated_count == 1
assert tag_and_push_release_service.git_api.push_count == 1

View file

@ -1,8 +1,9 @@
import os import os
from pybuilder.core import Project from pybuilder.core import Project
from src.main.python.ddadevops.c4k_mixin import C4kMixin, add_c4k_mixin_config from src.main.python.ddadevops.domain import DnsRecord
from src.main.python.ddadevops.c4k_mixin import C4kBuild, add_c4k_mixin_config
class MyC4kMixin(C4kMixin): class MyC4kBuild(C4kBuild):
pass pass
def test_c4k_mixin(tmp_path): def test_c4k_mixin(tmp_path):
@ -24,25 +25,22 @@ def test_c4k_mixin(tmp_path):
config = {'a': 1, 'b': 2} config = {'a': 1, 'b': 2}
auth = {'c': 3, 'd': 4} auth = {'c': 3, 'd': 4}
add_c4k_mixin_config(project_config, module_name, config, auth, grafana_cloud_user='user', grafana_cloud_password='password') add_c4k_mixin_config(project_config, config, auth, grafana_cloud_user='user', grafana_cloud_password='password')
assert project_config.get('C4kMixin') is not None assert project_config.get('C4kMixin') is not None
assert project_config.get('C4kMixin').get('Name') is module_name
assert project_config.get('C4kMixin').get('Config') is config
assert project_config.get('C4kMixin').get('Auth') is auth
mixin = MyC4kMixin(project, project_config) mixin = MyC4kBuild(project, project_config)
mixin.initialize_build_dir() mixin.initialize_build_dir()
assert mixin.build_path() == f'{tmp_path_str}/{build_dir}/{project_name}/{module_name}' assert mixin.build_path() == f'{tmp_path_str}/{build_dir}/{project_name}/{module_name}'
mixin.put('fqdn', 'testing.test') mixin.update_runtime_config(DnsRecord('test.de', ipv6="1::"))
sut = mixin.repo.get_c4k(mixin.project)
assert 'fqdn' in sut.config()
assert 'mon-cfg' in sut.config()
assert 'mon-auth' in sut.c4k_mixin_auth
mixin.write_c4k_config() mixin.write_c4k_config()
assert 'fqdn' in mixin.c4k_mixin_config
assert 'mon-cfg' in mixin.c4k_mixin_config
assert os.path.exists(f'{mixin.build_path()}/out_c4k_config.yaml') assert os.path.exists(f'{mixin.build_path()}/out_c4k_config.yaml')
mixin.write_c4k_auth() mixin.write_c4k_auth()
assert 'mon-auth' in mixin.c4k_mixin_auth
assert os.path.exists(f'{mixin.build_path()}/out_c4k_auth.yaml') assert os.path.exists(f'{mixin.build_path()}/out_c4k_auth.yaml')

View file

@ -0,0 +1,27 @@
import os
from pybuilder.core import Project
from src.main.python.ddadevops.domain.common import Devops
from src.main.python.ddadevops.devops_build import DevopsBuild
class MyDevopsBuild(DevopsBuild):
pass
def test_devops_build(tmp_path):
build_dir = "build"
project_name = "testing-project"
module_name = "c4k-test"
tmp_path_str = str(tmp_path)
project = Project(tmp_path_str, name=project_name)
devops = Devops(
stage="test",
project_root_path=tmp_path_str,
module=module_name,
build_dir_name=build_dir,
)
devops_build = DevopsBuild(project, devops=devops)
devops_build.initialize_build_dir()
assert os.path.exists(f"{devops_build.build_path()}")

View file

@ -0,0 +1,25 @@
import os
from pybuilder.core import Project
from src.main.python.ddadevops.domain import Image, Devops
from src.main.python.ddadevops.devops_image_build import DevopsImageBuild
def test_devops_docker_build(tmp_path):
build_dir = "build"
project_name = "testing-project"
module_name = "docker-test"
tmp_path_str = str(tmp_path)
project = Project(tmp_path_str, name=project_name)
devops = Devops(
stage="test",
project_root_path=tmp_path_str,
module=module_name,
name=project_name,
build_dir_name=build_dir
)
image = Image(dockerhub_user="user", dockerhub_password="password", devops=devops)
docker_build = DevopsImageBuild(project, image=image)
# docker_build.initialize_build_dir()
# assert os.path.exists(f"{docker_build.build_path()}")

View file

@ -0,0 +1,5 @@
(defproject org.domaindrivenarchitecture/c4k-website "1.1.3"
:description "website c4k-installation package"
:url "https://domaindrivenarchitecture.org"
:license {:name "Apache License, Version 2.0"
:url "https://www.apache.org/licenses/LICENSE-2.0.html"})

View file

@ -0,0 +1,2 @@
version = "12.4.678"

View file

@ -0,0 +1,3 @@
{
"version": "123.123.456"
}

View file

@ -0,0 +1,78 @@
# dda_devops_build
# Copyright 2019 meissa GmbH.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pybuilder.core import init, use_plugin, Author
use_plugin("python.core")
use_plugin("copy_resources")
use_plugin("filter_resources")
#use_plugin("python.unittest")
#use_plugin("python.coverage")
use_plugin("python.distutils")
#use_plugin("python.install_dependencies")
default_task = "publish"
name = "ddadevops"
version = "3.1.3"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
url = "https://github.com/DomainDrivenArchitecture/dda-devops-build"
requires_python = ">=2.7,!=3.0,!=3.1,!=3.2,!=3.3,!=3.4" # CHECK IF NEW VERSION EXISTS
license = "Apache Software License"
@init
def initialize(project):
#project.build_depends_on('mockito')
#project.build_depends_on('unittest-xml-reporting')
project.set_property("verbose", True)
project.get_property("filter_resources_glob").append("main/python/ddadevops/__init__.py")
#project.set_property("dir_source_unittest_python", "src/unittest/python")
project.set_property("copy_resources_target", "$dir_dist/ddadevops")
project.get_property("copy_resources_glob").append("LICENSE")
project.get_property("copy_resources_glob").append("src/main/resources/terraform/*")
project.get_property("copy_resources_glob").append("src/main/resources/docker/image/resources/*")
project.include_file("ddadevops", "LICENSE")
project.include_file("ddadevops", "src/main/resources/terraform/*")
project.include_file("ddadevops", "src/main/resources/docker/image/resources/*")
#project.set_property('distutils_upload_sign', True)
#project.set_property('distutils_upload_sign_identity', '')
project.set_property("distutils_readme_description", True)
project.set_property("distutils_description_overwrite", True)
project.set_property("distutils_classifiers", [
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Operating System :: POSIX :: Linux',
'Operating System :: OS Independent',
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: Developers',
'License :: OSI Approved :: Apache Software License',
'Topic :: Software Development :: Build Tools',
'Topic :: Software Development :: Quality Assurance',
'Topic :: Software Development :: Testing'
])