Merge branch 'feature/improve-image-build' into 'main'

improve image names

See merge request domaindrivenarchitecture/dda-devops-build!17
This commit is contained in:
Michael Jerger 2023-07-14 11:35:49 +00:00
commit 7002683a84
22 changed files with 448 additions and 115 deletions

View file

@ -1,56 +1,74 @@
image: "domaindrivenarchitecture/devops-build:4.0.8"
services:
- docker:dind
before_script:
- python --version
- python -m pip install --upgrade pip
- pip install -r requirements.txt
- export IMAGE_TAG=$CI_COMMIT_TAG
- export IMAGE_DOCKERHUB_USER=$DOCKERHUB_USER
- export IMAGE_DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
stages:
- lint&test
- upload
- image
.py: &py
image: "domaindrivenarchitecture/ddadevops-python:latest"
before_script:
- python --version
- pip install -r requirements.txt
.img: &img
image: "domaindrivenarchitecture/ddadevops-dind:latest"
services:
- docker:dind
before_script:
- export IMAGE_DOCKERHUB_USER=$DOCKERHUB_USER
- export IMAGE_DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
- export IMAGE_TAG=$CI_COMMIT_TAG
.tag_only: &tag_only
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/'
lint:
<<: *py
stage: lint&test
script:
- pip install -r dev_requirements.txt
- pyb lint
pytest:
<<: *py
stage: lint&test
script:
- pip install -r dev_requirements.txt
- pyb test
pypi-stable:
<<: *py
<<: *tag_only
stage: upload
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/'
script:
- pyb -P version=$CI_COMMIT_TAG publish upload
clojure-image-test-publish:
clojure-image-publish:
<<: *img
<<: *tag_only
stage: image
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/'
script:
- cd infrastructure/clojure && pyb image test publish
- cd infrastructure/clojure && pyb image publish
devops-build-image-test-publish:
python-image-publish:
<<: *img
<<: *tag_only
stage: image
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/'
script:
- cd infrastructure/devops-build && pyb image test publish
- cd infrastructure/python && pyb image publish
dind-image-publish:
<<: *img
<<: *tag_only
stage: image
script:
- cd infrastructure/dind && pyb image publish
ddadevops-image-publish:
<<: *img
<<: *tag_only
stage: image
script:
- cd infrastructure/ddadevops && pyb image publish

View file

@ -186,7 +186,7 @@ def destroy(project):
pyb dev publish upload
pip3 install --upgrade ddadevops --pre
pyb [patch|minor|major] release
pyb [patch|minor|major]
pip3 install --upgrade ddadevops
```

139
build.py
View file

@ -22,63 +22,71 @@ from ddadevops import *
use_plugin("python.core")
use_plugin("copy_resources")
use_plugin("filter_resources")
#use_plugin("python.unittest")
#use_plugin("python.coverage")
# use_plugin("python.unittest")
# use_plugin("python.coverage")
use_plugin("python.distutils")
#use_plugin("python.install_dependencies")
# use_plugin("python.install_dependencies")
default_task = "dev"
name = "ddadevops"
MODULE = "not-used"
PROJECT_ROOT_PATH = "."
version = "4.0.17-dev"
version = "4.1.0-dev2"
summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud"
description = __doc__
authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")]
url = "https://repo.prod.meissa.de/meissa/dda-devops-build"
requires_python = ">=3.10" # CHECK IF NEW VERSION EXISTS
requires_python = ">=3.10" # CHECK IF NEW VERSION EXISTS
license = "Apache Software License"
@init
def initialize(project):
#project.build_depends_on('mockito')
#project.build_depends_on('unittest-xml-reporting')
# project.build_depends_on('mockito')
# project.build_depends_on('unittest-xml-reporting')
project.build_depends_on("ddadevops>=4.0.0")
project.set_property("verbose", True)
project.get_property("filter_resources_glob").append("main/python/ddadevops/__init__.py")
project.get_property("filter_resources_glob").append(
"main/python/ddadevops/__init__.py"
)
project.set_property("dir_source_unittest_python", "src/test/python")
project.set_property("copy_resources_target", "$dir_dist/ddadevops")
project.get_property("copy_resources_glob").append("LICENSE")
project.get_property("copy_resources_glob").append("src/main/resources/terraform/*")
project.get_property("copy_resources_glob").append("src/main/resources/docker/image/resources/*")
project.get_property("copy_resources_glob").append(
"src/main/resources/docker/image/resources/*"
)
project.include_file("ddadevops", "LICENSE")
project.include_file("ddadevops", "src/main/resources/terraform/*")
project.include_file("ddadevops", "src/main/resources/docker/image/resources/*")
#project.set_property('distutils_upload_sign', True)
#project.set_property('distutils_upload_sign_identity', '')
# project.set_property('distutils_upload_sign', True)
# project.set_property('distutils_upload_sign_identity', '')
project.set_property("distutils_readme_description", True)
project.set_property("distutils_description_overwrite", True)
project.set_property("distutils_classifiers", [
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.10',
'Operating System :: POSIX :: Linux',
'Operating System :: OS Independent',
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: Developers',
'License :: OSI Approved :: Apache Software License',
'Topic :: Software Development :: Build Tools',
'Topic :: Software Development :: Quality Assurance',
'Topic :: Software Development :: Testing'
])
project.set_property(
"distutils_classifiers",
[
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.10",
"Operating System :: POSIX :: Linux",
"Operating System :: OS Independent",
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Developers",
"License :: OSI Approved :: Apache Software License",
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Quality Assurance",
"Topic :: Software Development :: Testing",
],
)
input = {
"name": name,
@ -88,54 +96,93 @@ def initialize(project):
"build_types": [],
"mixin_types": ["RELEASE"],
"release_primary_build_file": "build.py",
"release_secondary_build_files": [
"infrastructure/python/build.py",
"infrastructure/dind/build.py",
"infrastructure/ddadevops/build.py",
"infrastructure/clojure/build.py",
],
}
build = ReleaseMixin(project, input)
build.initialize_build_dir()
@task
def test(project):
run("pytest", check=True)
@task
def lint(project):
run("flake8 --max-line-length=120 --count --select=E9,F63,F7,F82 "+
"--show-source --statistics src/main/python/ddadevops/", shell=True, check=True)
run("flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 "+
"--per-file-ignores=\"__init__.py:F401\" "+
"--ignore=E722,W503 --statistics src/main/python/ddadevops/", shell=True, check=True)
run("python -m mypy src/main/python/ddadevops/ --ignore-missing-imports "+
"--disable-error-code=attr-defined --disable-error-code=union-attr", shell=True, check=True)
run("pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0702,W0702,"+
"R0913,R0902,R0914,R1732,R1705,W0707,C0123,W0703,C0103 src/main/python/ddadevops/", shell=True, check=True)
run(
"flake8 --max-line-length=120 --count --select=E9,F63,F7,F82 "
+ "--show-source --statistics src/main/python/ddadevops/",
shell=True,
check=True,
)
run(
"flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 "
+ '--per-file-ignores="__init__.py:F401" '
+ "--ignore=E722,W503 --statistics src/main/python/ddadevops/",
shell=True,
check=True,
)
run(
"python -m mypy src/main/python/ddadevops/ --ignore-missing-imports "
+ "--disable-error-code=attr-defined --disable-error-code=union-attr",
shell=True,
check=True,
)
run(
"pylint -d W0511,R0903,C0301,W0614,C0114,C0115,C0116,similarities,W1203,W0702,W0702,"
+ "R0913,R0902,R0914,R1732,R1705,W0707,C0123,W0703,C0103 src/main/python/ddadevops/",
shell=True,
check=True,
)
@task
def patch(project):
build(project, "PATCH")
linttest(project, "PATCH")
release(project)
@task
def minor(project):
build(project, "MINOR")
linttest(project, "MINOR")
release(project)
@task
def major(project):
build(project, "MAJOR")
linttest(project, "MAJOR")
release(project)
@task
def dev(project):
build(project, "NONE")
linttest(project, "NONE")
@task
def nothing(project):
pass
@task
def release(project):
def prepare(project):
build = get_devops_build(project)
build.prepare_release()
@task
def tag(project):
build = get_devops_build(project)
build.tag_bump_and_push_release()
def build(project, release_type):
def release(project):
prepare(project)
tag(project)
def linttest(project, release_type):
build = get_devops_build(project)
build.update_release_type(release_type)
test(project)

33
doc/Images.md Normal file
View file

@ -0,0 +1,33 @@
# ddadevops Images
## ddadevops-clojure
Contains
* clojure
* shadowcljs
* lein
* java
* graalvm
* pybuilder, ddadevops
## ddadevops
Contains:
* pybuilder, ddadevops
## devops-build
Image is deprecated.
## ddadevops-dind
Contains:
* docker in docker
* pybuilder, ddadevops
## ddadevops-python
Contains:
* python 3.10
* python linting
* python setup-tools
* pybuilder, ddadevops

View file

@ -12,6 +12,7 @@ classDiagram
}
class Image {
image_naming
image_dockerhub_user
image_dockerhub_password
image_publish_tag

View file

@ -1,8 +0,0 @@
adjust version no in build.py to release version no.
git commit -am "release"
git tag -am "release" [release version no]
git push --follow-tags
increase version no in build.py
git commit -am "version bump"
git push
pip3 install --upgrade ddadevops

View file

@ -1,14 +1,18 @@
from os import environ
from datetime import datetime
from pybuilder.core import task, init
from ddadevops import *
name = "clojure"
MODULE = "image"
name = "ddadevops"
MODULE = "clojure"
PROJECT_ROOT_PATH = "../.."
version = "4.1.0-dev"
@init
def initialize(project):
image_tag = version
if "dev" in image_tag:
image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
input = {
"name": name,
@ -17,6 +21,8 @@ def initialize(project):
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["IMAGE"],
"mixin_types": [],
"image_naming": "NAME_AND_MODULE",
"image_tag": f"{image_tag}",
}
project.build_depends_on("ddadevops>=4.0.0")

View file

@ -0,0 +1,57 @@
from os import environ
from datetime import datetime
from pybuilder.core import task, init
from ddadevops import *
name = "ddadevops"
MODULE = "ddadevops"
PROJECT_ROOT_PATH = "../.."
version = "4.1.0-dev"
@init
def initialize(project):
image_tag = version
if "dev" in image_tag:
image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
input = {
"name": name,
"module": MODULE,
"stage": "notused",
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["IMAGE"],
"mixin_types": [],
"image_naming": "NAME_ONLY",
"image_tag": f"{image_tag}",
}
project.build_depends_on("ddadevops>=4.0.0")
build = DevopsImageBuild(project, input)
build.initialize_build_dir()
@task
def image(project):
build = get_devops_build(project)
build.image()
@task
def drun(project):
build = get_devops_build(project)
build.drun()
@task
def test(project):
build = get_devops_build(project)
build.test()
@task
def publish(project):
build = get_devops_build(project)
build.dockerhub_login()
build.dockerhub_publish()

View file

@ -0,0 +1,6 @@
FROM python:3.10-alpine
RUN set -eux;
RUN apk add --no-cache python3 py3-pip openssl-dev bash git;
RUN python3 -m pip install -U pip;
RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection;

View file

View file

@ -5,6 +5,7 @@ from ddadevops import *
name = "devops-build"
MODULE = "image"
PROJECT_ROOT_PATH = "../.."
version = "4.0.0-dev73"
@init

View file

@ -0,0 +1,57 @@
from os import environ
from datetime import datetime
from pybuilder.core import task, init
from ddadevops import *
name = "ddadevops"
MODULE = "dind"
PROJECT_ROOT_PATH = "../.."
version = "4.1.0-dev"
@init
def initialize(project):
image_tag = version
if "dev" in image_tag:
image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
input = {
"name": name,
"module": MODULE,
"stage": "notused",
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["IMAGE"],
"mixin_types": [],
"image_naming": "NAME_AND_MODULE",
"image_tag": f"{image_tag}",
}
project.build_depends_on("ddadevops>=4.0.0")
build = DevopsImageBuild(project, input)
build.initialize_build_dir()
@task
def image(project):
build = get_devops_build(project)
build.image()
@task
def drun(project):
build = get_devops_build(project)
build.drun()
@task
def test(project):
build = get_devops_build(project)
build.test()
@task
def publish(project):
build = get_devops_build(project)
build.dockerhub_login()
build.dockerhub_publish()

View file

@ -0,0 +1,6 @@
FROM docker:latest
RUN set -eux;
RUN apk add --no-cache python3 py3-pip openssl-dev bash git;
RUN python3 -m pip install -U pip;
RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection;

View file

View file

@ -0,0 +1,57 @@
from os import environ
from datetime import datetime
from pybuilder.core import task, init
from ddadevops import *
name = "ddadevops"
MODULE = "python"
PROJECT_ROOT_PATH = "../.."
version = "4.1.0-dev"
@init
def initialize(project):
image_tag = version
if "dev" in image_tag:
image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
input = {
"name": name,
"module": MODULE,
"stage": "notused",
"project_root_path": PROJECT_ROOT_PATH,
"build_types": ["IMAGE"],
"mixin_types": [],
"image_naming": "NAME_AND_MODULE",
"image_tag": f"{image_tag}",
}
project.build_depends_on("ddadevops>=4.0.0")
build = DevopsImageBuild(project, input)
build.initialize_build_dir()
@task
def image(project):
build = get_devops_build(project)
build.image()
@task
def drun(project):
build = get_devops_build(project)
build.drun()
@task
def test(project):
build = get_devops_build(project)
build.test()
@task
def publish(project):
build = get_devops_build(project)
build.dockerhub_login()
build.dockerhub_publish()

View file

@ -0,0 +1,7 @@
FROM python:3.10-alpine
RUN set -eux;
RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptools py3-wheel libffi-dev openssl-dev cargo bash git;
RUN python3 -m pip install -U pip;
RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection;
RUN pip3 install coverage flake8 flake8-polyfill mypy mypy-extensions pycodestyle pyflakes pylint pytest pytest-cov pytest-datafiles types-setuptools types-PyYAML;

View file

View file

@ -4,7 +4,9 @@ from ..infrastructure import FileApi, ResourceApi, ImageApi
class ImageBuildService:
def __init__(self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi):
def __init__(
self, file_api: FileApi, resource_api: ResourceApi, image_api: ImageApi
):
self.file_api = file_api
self.resource_api = resource_api
self.image_api = image_api
@ -18,7 +20,9 @@ class ImageBuildService:
)
def __copy_build_resource_file_from_package__(self, resource_name, devops: Devops):
data = self.resource_api.read_resource(f"src/main/resources/docker/{resource_name}")
data = self.resource_api.read_resource(
f"src/main/resources/docker/{resource_name}"
)
self.file_api.write_data_to_file(
Path(f"{devops.build_path()}/{resource_name}"), data
)
@ -30,9 +34,7 @@ class ImageBuildService:
def __copy_build_resources_from_dir__(self, devops: Devops):
image = devops.specialized_builds[BuildType.IMAGE]
self.file_api.cp_force(
image.build_commons_path(), devops.build_path()
)
self.file_api.cp_force(image.build_commons_path(), devops.build_path())
def initialize_build_dir(self, devops: Devops):
image = devops.specialized_builds[BuildType.IMAGE]
@ -46,10 +48,12 @@ class ImageBuildService:
self.file_api.cp_recursive("test", build_path)
def image(self, devops: Devops):
self.image_api.image(devops.name, devops.build_path())
image = devops.specialized_builds[BuildType.IMAGE]
self.image_api.image(image.image_name(), devops.build_path())
def drun(self, devops: Devops):
self.image_api.drun(devops.name)
image = devops.specialized_builds[BuildType.IMAGE]
self.image_api.drun(image.image_name())
def dockerhub_login(self, devops: Devops):
image = devops.specialized_builds[BuildType.IMAGE]
@ -59,9 +63,14 @@ class ImageBuildService:
def dockerhub_publish(self, devops: Devops):
image = devops.specialized_builds[BuildType.IMAGE]
if image.image_tag is not None:
self.image_api.dockerhub_publish(
image.image_name(), image.image_dockerhub_user, image.image_tag
)
self.image_api.dockerhub_publish(
devops.name, image.image_dockerhub_user, image.image_tag
image.image_name(), image.image_dockerhub_user, 'latest'
)
def test(self, devops: Devops):
self.image_api.test(devops.name, devops.build_path())
image = devops.specialized_builds[BuildType.IMAGE]
self.image_api.test(image.image_name(), devops.build_path())

View file

@ -1,3 +1,4 @@
from enum import Enum
from typing import List, Dict
from .common import (
filter_none,
@ -5,15 +6,23 @@ from .common import (
)
class NamingType(Enum):
NAME_ONLY = 1
NAME_AND_MODULE = 2
class Image(Validateable):
def __init__(
self,
inp: dict,
):
self.module = inp.get("module")
self.name = inp.get("name")
self.image_dockerhub_user = inp.get("image_dockerhub_user")
self.image_dockerhub_password = inp.get("image_dockerhub_password")
self.image_tag = inp.get("image_tag")
self.image_build_commons_path = inp.get("image_build_commons_path")
self.image_naming = NamingType[inp.get("image_naming", "NAME_ONLY")]
self.image_use_package_common_files = inp.get(
"image_use_package_common_files", True
)
@ -23,8 +32,10 @@ class Image(Validateable):
def validate(self) -> List[str]:
result = []
result += self.__validate_is_not_empty__("name")
result += self.__validate_is_not_empty__("image_dockerhub_user")
result += self.__validate_is_not_empty__("image_dockerhub_password")
result += self.__validate_is_not_empty__("image_naming")
if not self.image_use_package_common_files:
result += self.__validate_is_not_empty__("image_build_commons_path")
result += self.__validate_is_not_empty__("image_build_commons_dir_name")
@ -37,6 +48,16 @@ class Image(Validateable):
]
return "/".join(filter_none(commons_path)) + "/"
def image_name(self) -> str:
result: List[str] = [self.name] # type: ignore
if (
self.image_naming == NamingType.NAME_AND_MODULE
and self.module
and self.module != ""
):
result.append(self.module)
return "-".join(result)
@classmethod
def get_mapping_default(cls) -> List[Dict[str, str]]:
return [

View file

@ -53,37 +53,30 @@ class ImageApi:
self.execution_api = ExecutionApi()
def image(self, name: str, path: Path):
self.execution_api.execute_handled(
self.execution_api.execute_live(
f"docker build -t {name} --file {path}/image/Dockerfile {path}/image"
)
def drun(self, name: str):
self.execution_api.execute_handled(
self.execution_api.execute_live(
f'docker run -it --entrypoint="" {name} /bin/bash'
)
def dockerhub_login(self, username: str, password: str):
self.execution_api.execute_handled(
self.execution_api.execute_live(
f"docker login --username {username} --password {password}"
)
def dockerhub_publish(self, name: str, username: str, tag=None):
if tag is not None:
self.execution_api.execute_handled(
f"docker tag {name} {username}/{name}:{tag}"
)
self.execution_api.execute_handled(
f"docker push {username}/{name}:{tag}"
)
self.execution_api.execute_handled(
f"docker tag {name} {username}/{name}:latest"
def dockerhub_publish(self, name: str, username: str, tag: str):
self.execution_api.execute_live(
f"docker tag {name} {username}/{name}:{tag}"
)
self.execution_api.execute_handled(
f"docker push {username}/{name}:latest"
self.execution_api.execute_live(
f"docker push {username}/{name}:{tag}"
)
def test(self, name: str, path: Path):
self.execution_api.execute_handled(
self.execution_api.execute_live(
f"docker build -t {name} -test --file {path}/test/Dockerfile {path}/test"
)
@ -101,6 +94,9 @@ class ExecutionApi:
output = output.rstrip()
return output
# TODO: check for exception handling
# TODO: can we return the output here also?
# TODO: should we also print stderr?
def execute_live(self, command: str, dry_run=False, shell=True):
if dry_run:
print(command)
@ -111,6 +107,7 @@ class ExecutionApi:
process.stdout.close()
process.wait()
# TODO: move this enhancement to execute
def execute_handled(self, command: str, dry_run=False, shell=True, check=True):
if dry_run:
print(command)

View file

@ -7,7 +7,7 @@ from src.main.python.ddadevops.domain import (
)
def test_devops_factory():
def test_devops_creation():
with pytest.raises(Exception):
DevopsFactory().build_devops({"build_types": ["NOTEXISTING"]})
@ -66,3 +66,8 @@ def test_devops_factory():
)
assert sut is not None
assert sut.mixins[MixinType.RELEASE] is not None
def test_on_merge_input_should_win():
sut = DevopsFactory()
assert {'tag': 'inp'} == sut.merge(inp = {'tag': 'inp'}, context = {'tag': 'context'}, authorization={})

View file

@ -12,3 +12,16 @@ def test_devops_build_commons_path():
assert image is not None
assert image.is_valid()
assert "docker/" == image.build_commons_path()
def test_should_calculate_image_name():
sut = build_devops({})
image = sut.specialized_builds[BuildType.IMAGE]
assert "name" == image.image_name()
sut = build_devops({'image_naming': "NAME_ONLY"})
image = sut.specialized_builds[BuildType.IMAGE]
assert "name" == image.image_name()
sut = build_devops({'image_naming': "NAME_AND_MODULE"})
image = sut.specialized_builds[BuildType.IMAGE]
assert "name-module" == image.image_name()