meissa/dda-devops-build
6
1
Fork 0
Code Issues Pull requests Projects Releases 33 Packages Wiki Activity

rename docker -> image & introduce domain sub-ns

Browse source
This commit is contained in:
Michael Jerger 2023-03-17 17:27:36 +01:00
parent a4fbeaadd9
commit 325c9d477f
20 changed files with 230 additions and 210 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -165,7 +165,7 @@ def access(project):
build.get_mfa_session()
```
## Feature DdaDockerBuild
## Feature DdaImageBuild
The docker build supports image building, tagging, testing and login to dockerhost.
For bash based builds we support often used script-parts as predefined functions [see install_functions.sh](src/main/resources/docker/image/resources/install_functions.sh).

2
build.py
View file

@ -28,7 +28,7 @@ use_plugin("python.distutils")
default_task = "publish"
name = "ddadevops"
version = "4.0.0-dev11"
version = "4.0.0-dev16"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]

4
doc/architecture/Build.md
View file

@ -54,7 +54,7 @@ classDiagram
init_client(self)
}
class DevopsDockerBuild {
class DevopsImageBuild {
def initialize_build_dir()
image()
drun()
@ -77,7 +77,7 @@ classDiagram
c4k_apply(dry_run=False)
}
DevopsBuild <|-- DevopsDockerBuild
DevopsBuild <|-- DevopsImageBuild
DevopsBuild <|-- DevopsTerraformBuild
DevopsBuild <|-- AwsRdsPgMixin

4
doc/architecture/Domain.md
View file

@ -10,7 +10,7 @@ classDiagram
build_dir_name
}
class Docker {
class Image {
dockerhub_user
dockerhub_password
build_dir_name
@ -33,7 +33,7 @@ classDiagram
}
C4k *-- DnsRecord
Docker *-- Devops
Image *-- Devops
```

2
doc/example/50_docker_module/build.py
View file

@ -5,7 +5,7 @@ name = 'example-project'
MODULE = 'docker-module'
PROJECT_ROOT_PATH = '../../..'
class MyBuild(DevopsDockerBuild):
class MyBuild(DevopsImageBuild):
pass
@init

6
infrastructure/clojure/build.py
View file

@ -1,8 +1,6 @@
from subprocess import run
from os import environ
from pybuilder.core import task, init
from ddadevops import *
import logging
name = "clojure"
MODULE = "docker"
@ -29,14 +27,14 @@ def initialize(project):
module=MODULE,
name=name,
)
docker = Docker(
image = Image(
dockerhub_user=dockerhub_user,
dockerhub_password=dockerhub_password,
docker_publish_tag=tag,
devops=devops,
)
build = DevopsDockerBuild(project, docker=docker)
build = DevopsImageBuild(project, image=image)
build.initialize_build_dir()

46
infrastructure/devops-build/build.py
View file

@ -1,33 +1,40 @@
from subprocess import run
from os import environ
from pybuilder.core import task, init
from ddadevops import *
import logging
name = 'devops-build'
MODULE = 'docker'
PROJECT_ROOT_PATH = '../..'
name = "devops-build"
MODULE = "docker"
PROJECT_ROOT_PATH = "../.."
class MyBuild(DevopsDockerBuild):
pass
@init
def initialize(project):
project.build_depends_on('ddadevops>=0.13.0')
stage = 'notused'
dockerhub_user = environ.get('DOCKERHUB_USER')
project.build_depends_on("ddadevops>=4.0.0-dev")
stage = "notused"
dockerhub_user = environ.get("DOCKERHUB_USER")
if not dockerhub_user:
dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login')
dockerhub_password = environ.get('DOCKERHUB_PASSWORD')
dockerhub_user = gopass_field_from_path("meissa/web/docker.com", "login")
dockerhub_password = environ.get("DOCKERHUB_PASSWORD")
if not dockerhub_password:
dockerhub_password = gopass_password_from_path('meissa/web/docker.com')
tag = environ.get('CI_COMMIT_TAG')
dockerhub_password = gopass_password_from_path("meissa/web/docker.com")
tag = environ.get("CI_COMMIT_TAG")
if not tag:
tag = get_tag_from_latest_commit()
config = create_devops_docker_build_config(
stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag)
build = MyBuild(project, config)
devops = Devops(
stage=stage,
project_root_path=PROJECT_ROOT_PATH,
module=MODULE,
name=name,
)
image = Image(
dockerhub_user=dockerhub_user,
dockerhub_password=dockerhub_password,
docker_publish_tag=tag,
devops=devops,
)
build = DevopsImageBuild(project, image=image)
build.initialize_build_dir()
@ -36,16 +43,19 @@ def image(project):
build = get_devops_build(project)
build.image()
@task
def drun(project):
build = get_devops_build(project)
build.drun()
@task
def test(project):
build = get_devops_build(project)
build.test()
@task
def publish(project):
build = get_devops_build(project)

6
src/main/python/ddadevops/__init__.py
View file

@ -14,13 +14,11 @@ from .exoscale_mixin import ExoscaleMixin, add_exoscale_mixin_config
from .digitalocean_backend_properties_mixin import DigitaloceanBackendPropertiesMixin, add_digitalocean_backend_properties_mixin_config
from .digitalocean_terraform_build import DigitaloceanTerraformBuild, create_digitalocean_terraform_build_config
from .hetzner_mixin import HetznerMixin, add_hetzner_mixin_config
from .devops_docker_build import DevopsDockerBuild, create_devops_docker_build_config
from .devops_image_build import DevopsImageBuild, create_devops_docker_build_config
from .devops_terraform_build import DevopsTerraformBuild, create_devops_terraform_build_config
from .devops_build import DevopsBuild, create_devops_build_config, get_devops_build, get_tag_from_latest_commit
from .credential import gopass_password_from_path, gopass_field_from_path
from .domain import Validateable, DnsRecord, Devops, Docker, C4k
from .application import DockerBuildService
from .infrastructure import ProjectRepository, ResourceApi, FileApi, DockerApi, ExecutionApi
from .domain import Validateable, DnsRecord, Devops, Image
__version__ = "${version}"

24
src/main/python/ddadevops/application.py
View file

@ -1,30 +1,30 @@
from .domain import Devops, Docker
from .infrastructure import FileApi, ResourceApi, DockerApi, ExecutionApi
from .domain import Devops, Image
from .infrastructure import FileApi, ResourceApi, ImageApi, ExecutionApi
class DockerBuildService:
class ImageBuildService:
def __init__(self):
self.file_api = FileApi()
self.resource_api = ResourceApi()
self.docker_api = DockerApi()
self.docker_api = ImageApi()
def __copy_build_resource_file_from_package__(self, resource_name, docker: Docker):
def __copy_build_resource_file_from_package__(self, resource_name, docker: Image):
data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}")
self.file_api.write_data_to_file(
f"{docker.devops.build_path()}/{resource_name}", data
)
def __copy_build_resources_from_package__(self, docker: Docker):
def __copy_build_resources_from_package__(self, docker: Image):
self.__copy_build_resource_file_from_package__(
"image/resources/install_functions.sh", docker
)
def __copy_build_resources_from_dir__(self, docker: Docker):
def __copy_build_resources_from_dir__(self, docker: Image):
self.file_api.cp_force(
docker.docker_build_commons_path(), docker.devops.build_path()
)
def initialize_build_dir(self, docker: Docker):
def initialize_build_dir(self, docker: Image):
build_path = docker.devops.build_path()
self.file_api.clean_dir(f"{build_path}/image/resources")
if docker.use_package_common_files:
@ -34,21 +34,21 @@ class DockerBuildService:
self.file_api.cp_recursive("image", build_path)
self.file_api.cp_recursive("test", build_path)
def image(self, docker: Docker):
def image(self, docker: Image):
self.docker_api.image(docker.devops.name, docker.devops.build_path())
def drun(self, docker: Devops):
self.docker_api.drun(docker.devops.name)
def dockerhub_login(self, docker: Docker):
def dockerhub_login(self, docker: Image):
self.docker_api.dockerhub_login(
docker.dockerhub_user, docker.dockerhub_password
)
def dockerhub_publish(self, docker: Docker):
def dockerhub_publish(self, docker: Image):
self.docker_api.dockerhub_publish(
docker.devops.name, docker.dockerhub_user, docker.docker_publish_tag
)
def test(self, docker: Docker):
def test(self, docker: Image):
self.docker_api.test(docker.devops.name, docker.devops.build_path())

42
src/main/python/ddadevops/devops_docker_build.py → src/main/python/ddadevops/devops_image_build.py
View file

@ -1,6 +1,6 @@
import deprecation
from .domain import Docker
from .application import DockerBuildService
from .domain import Image
from .application import ImageBuildService
from .devops_build import DevopsBuild, create_devops_build_config
@ -31,11 +31,11 @@ def create_devops_docker_build_config(
return ret
class DevopsDockerBuild(DevopsBuild):
def __init__(self, project, config: map = None, docker: Docker = None):
self.docker_build_service = DockerBuildService()
if not docker:
docker = Docker(
class DevopsImageBuild(DevopsBuild):
def __init__(self, project, config: map = None, image: Image = None):
self.image_build_service = ImageBuildService()
if not image:
image = Image(
dockerhub_user=config["dockerhub_user"],
dockerhub_password=config["dockerhub_password"],
use_package_common_files=config["use_package_common_files"],
@ -45,30 +45,30 @@ class DevopsDockerBuild(DevopsBuild):
)
super().__init__(project, config=config)
else:
super().__init__(project, devops=docker.devops)
self.repo.set_docker(self.project, docker)
super().__init__(project, devops=image.devops)
self.repo.set_docker(self.project, image)
def initialize_build_dir(self):
super().initialize_build_dir()
docker = self.repo.get_docker(self.project)
self.docker_build_service.initialize_build_dir(docker)
image = self.repo.get_docker(self.project)
self.image_build_service.initialize_build_dir(image)
def image(self):
docker = self.repo.get_docker(self.project)
self.docker_build_service.image(docker)
image = self.repo.get_docker(self.project)
self.image_build_service.image(image)
def drun(self):
docker = self.repo.get_docker(self.project)
self.docker_build_service.drun(docker)
image = self.repo.get_docker(self.project)
self.image_build_service.drun(image)
def dockerhub_login(self):
docker = self.repo.get_docker(self.project)
self.docker_build_service.dockerhub_login(docker)
image = self.repo.get_docker(self.project)
self.image_build_service.dockerhub_login(image)
def dockerhub_publish(self):
docker = self.repo.get_docker(self.project)
self.docker_build_service.dockerhub_publish(docker)
image = self.repo.get_docker(self.project)
self.image_build_service.dockerhub_publish(image)
def test(self):
docker = self.repo.get_docker(self.project)
self.docker_build_service.test(docker)
image = self.repo.get_docker(self.project)
self.image_build_service.test(image)

131
src/main/python/ddadevops/domain.py
View file

@ -1,131 +0,0 @@
import deprecation
from typing import List
from .python_util import filter_none
class Validateable:
def __validate_is_not_empty__(self, field_name: str) -> List[str]:
value = self.__dict__[field_name]
if value is None or value == "":
return [f"Field '{field_name}' may not be empty."]
else:
return []
def validate(self) -> List[str]:
return []
def is_valid(self) -> bool:
return len(self.validate()) < 1
class DnsRecord(Validateable):
def __init__(self, fqdn, ipv4=None, ipv6=None):
self.fqdn = fqdn
self.ipv4 = ipv4
self.ipv6 = ipv6
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("fqdn")
if (not self.ipv4) and (not self.ipv6):
result.append("ipv4 & ipv6 may not both be empty.")
return result
class Devops(Validateable):
def __init__(
self, stage, project_root_path, module, name=None, build_dir_name="target"
):
self.stage = stage
self.name = name
self.project_root_path = project_root_path
self.module = module
if not name:
self.name = module
self.build_dir_name = build_dir_name
# Deprecated - no longer use generic stack ...
self.stack = {}
@deprecation.deprecated(deprecated_in="3.2")
# use .name instead
def name(self):
return self.name
def build_path(self):
path = [self.project_root_path, self.build_dir_name, self.name, self.module]
return "/".join(filter_none(path))
def __put__(self, key, value):
self.stack[key] = value
def __get__(self, key):
return self.stack[key]
def __get_keys__(self, keys):
result = {}
for key in keys:
result[key] = self.__get__(key)
return result
class Docker(Validateable):
def __init__(
self,
dockerhub_user,
dockerhub_password,
devops: Devops,
build_dir_name="target",
use_package_common_files=True,
build_commons_path=None,
docker_build_commons_dir_name="docker",
docker_publish_tag=None,
):
self.dockerhub_user = dockerhub_user
self.dockerhub_password = dockerhub_password
self.use_package_common_files = use_package_common_files
self.build_commons_path = build_commons_path
self.docker_build_commons_dir_name = docker_build_commons_dir_name
self.docker_publish_tag = docker_publish_tag
self.devops = devops
def docker_build_commons_path(self):
list = [self.build_commons_path, self.docker_build_commons_dir_name]
return "/".join(filter_none(list)) + "/"
class C4k(Validateable):
def __init__(self, config: map):
tmp_executabel_name = config["C4kMixin"]["executabel_name"]
if not tmp_executabel_name:
tmp_executabel_name = config["module"]
self.executabel_name = tmp_executabel_name
self.c4k_mixin_config = config["C4kMixin"]["config"]
self.c4k_mixin_auth = config["C4kMixin"]["auth"]
tmp = self.c4k_mixin_config["mon-cfg"]
tmp.update({"cluster-name": config["module"], "cluster-stage": config["stage"]})
self.c4k_mixin_config.update({"mon-cfg": tmp})
self.dns_record = None
# TODO: these functions should be located at TerraformBuild later on.
def update_runtime_config(self, dns_record: DnsRecord):
self.dns_record = dns_record
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("fqdn")
if self.dns_record:
result += self.dns_record.validate()
return result
def config(self):
fqdn = self.dns_record.fqdn
self.c4k_mixin_config.update({"fqdn": fqdn})
return self.c4k_mixin_config
def command(self, build: Devops):
module = build.module
build_path = build.build_path()
config_path = f"{build_path}/out_c4k_config.yaml"
auth_path = f"{build_path}/out_c4k_auth.yaml"
output_path = f"{build_path}/out_{module}.yaml"
return f"c4k-{self.executabel_name}-standalone.jar {config_path} {auth_path} > {output_path}"

3
src/main/python/ddadevops/domain/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from .common import Validateable, DnsRecord, Devops
from .image import Image
from .c4k import C4k

44
src/main/python/ddadevops/domain/c4k.py Normal file
View file

@ -0,0 +1,44 @@
import deprecation
from typing import List
from .common import (
Validateable,
DnsRecord,
Devops,
)
class C4k(Validateable):
def __init__(self, config: map):
tmp_executabel_name = config["C4kMixin"]["executabel_name"]
if not tmp_executabel_name:
tmp_executabel_name = config["module"]
self.executabel_name = tmp_executabel_name
self.c4k_mixin_config = config["C4kMixin"]["config"]
self.c4k_mixin_auth = config["C4kMixin"]["auth"]
tmp = self.c4k_mixin_config["mon-cfg"]
tmp.update({"cluster-name": config["module"], "cluster-stage": config["stage"]})
self.c4k_mixin_config.update({"mon-cfg": tmp})
self.dns_record = None
# TODO: these functions should be located at TerraformBuild later on.
def update_runtime_config(self, dns_record: DnsRecord):
self.dns_record = dns_record
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("fqdn")
if self.dns_record:
result += self.dns_record.validate()
return result
def config(self):
fqdn = self.dns_record.fqdn
self.c4k_mixin_config.update({"fqdn": fqdn})
return self.c4k_mixin_config
def command(self, build: Devops):
module = build.module
build_path = build.build_path()
config_path = f"{build_path}/out_c4k_config.yaml"
auth_path = f"{build_path}/out_c4k_auth.yaml"
output_path = f"{build_path}/out_{module}.yaml"
return f"c4k-{self.executabel_name}-standalone.jar {config_path} {auth_path} > {output_path}"

69
src/main/python/ddadevops/domain/common.py Normal file
View file

@ -0,0 +1,69 @@
import deprecation
from typing import List
def filter_none(list_to_filter):
return [x for x in list_to_filter if x is not None]
class Validateable:
def __validate_is_not_empty__(self, field_name: str) -> List[str]:
value = self.__dict__[field_name]
if value is None or value == "":
return [f"Field '{field_name}' may not be empty."]
else:
return []
def validate(self) -> List[str]:
return []
def is_valid(self) -> bool:
return len(self.validate()) < 1
class DnsRecord(Validateable):
def __init__(self, fqdn, ipv4=None, ipv6=None):
self.fqdn = fqdn
self.ipv4 = ipv4
self.ipv6 = ipv6
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("fqdn")
if (not self.ipv4) and (not self.ipv6):
result.append("ipv4 & ipv6 may not both be empty.")
return result
class Devops(Validateable):
def __init__(
self, stage, project_root_path, module, name=None, build_dir_name="target"
):
self.stage = stage
self.name = name
self.project_root_path = project_root_path
self.module = module
if not name:
self.name = module
self.build_dir_name = build_dir_name
# Deprecated - no longer use generic stack ...
self.stack = {}
@deprecation.deprecated(deprecated_in="3.2")
# use .name instead
def name(self):
return self.name
def build_path(self):
path = [self.project_root_path, self.build_dir_name, self.name, self.module]
return "/".join(filter_none(path))
def __put__(self, key, value):
self.stack[key] = value
def __get__(self, key):
return self.stack[key]
def __get_keys__(self, keys):
result = {}
for key in keys:
result[key] = self.__get__(key)
return result

29
src/main/python/ddadevops/domain/image.py Normal file
View file

@ -0,0 +1,29 @@
from .common import (
filter_none,
Validateable,
Devops,
)
class Image(Validateable):
def __init__(
self,
dockerhub_user,
dockerhub_password,
devops: Devops,
build_dir_name="target",
use_package_common_files=True,
build_commons_path=None,
docker_build_commons_dir_name="docker",
docker_publish_tag=None,
):
self.dockerhub_user = dockerhub_user
self.dockerhub_password = dockerhub_password
self.use_package_common_files = use_package_common_files
self.build_commons_path = build_commons_path
self.docker_build_commons_dir_name = docker_build_commons_dir_name
self.docker_publish_tag = docker_publish_tag
self.devops = devops
def docker_build_commons_path(self):
list = [self.build_commons_path, self.docker_build_commons_dir_name]
return "/".join(filter_none(list)) + "/"

8
src/main/python/ddadevops/infrastructure.py
View file

@ -4,7 +4,7 @@ from pkg_resources import resource_string
from os import chmod
from subprocess import run
import yaml
from .domain import Devops, Docker, C4k
from .domain import Devops, Image, C4k
from .python_util import execute, execute_live
@ -18,10 +18,10 @@ class ProjectRepository:
def set_devops(self, project, build: Devops):
project.set_property("build", build)
def get_docker(self, project) -> Docker:
def get_docker(self, project) -> Image:
return project.get_property("docker_build")
def set_docker(self, project, build: Docker):
def set_docker(self, project, build: Image):
project.set_property("docker_build", build)
def get_c4k(self, project) -> C4k:
@ -57,7 +57,7 @@ class FileApi:
chmod(path, 0o600)
class DockerApi:
class ImageApi:
def image(self, name: str, path: Path):
run(
f"docker build -t {name} --file {path}/image/Dockerfile {path}/image",

0
src/test/python/domain/__init__.py Normal file
View file

8
src/test/python/test_domain.py → src/test/python/domain/test_domain.py
View file

@ -1,11 +1,11 @@
from pybuilder.core import Project
from src.main.python.ddadevops.domain import (
from src.main.python.ddadevops.domain.common import (
Validateable,
DnsRecord,
C4k,
Devops,
Docker,
)
from src.main.python.ddadevops.domain.image import Image
from src.main.python.ddadevops.domain.c4k import C4k
from src.main.python.ddadevops.c4k_mixin import add_c4k_mixin_config
@ -72,7 +72,7 @@ def test_devops_build_commons_path():
devops = Devops(
stage="test", project_root_path="../../..", module="cloud", name="meissa"
)
sut = Docker(
sut = Image(
dockerhub_user="user",
dockerhub_password="password",
devops = devops,

2
src/test/python/test_devops_build.py
View file

@ -1,6 +1,6 @@
import os
from pybuilder.core import Project
from src.main.python.ddadevops.domain import Devops
from src.main.python.ddadevops.domain.common import Devops
from src.main.python.ddadevops.devops_build import DevopsBuild

8
src/test/python/test_docker_build.py → src/test/python/test_image_build.py
View file

@ -1,7 +1,7 @@
import os
from pybuilder.core import Project
from src.main.python.ddadevops.domain import Docker, Devops
from src.main.python.ddadevops.devops_docker_build import DevopsDockerBuild
from src.main.python.ddadevops.domain import Image, Devops
from src.main.python.ddadevops.devops_image_build import DevopsImageBuild
def test_devops_docker_build(tmp_path):
@ -17,8 +17,8 @@ def test_devops_docker_build(tmp_path):
module=module_name,
name=project_name,
)
docker = Docker(dockerhub_user="user", dockerhub_password="password", devops=devops)
image = Image(dockerhub_user="user", dockerhub_password="password", devops=devops)
docker_build = DevopsDockerBuild(project, docker=docker)
docker_build = DevopsImageBuild(project, image=image)
# docker_build.initialize_build_dir()
# assert os.path.exists(f"{docker_build.build_path()}")