diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml deleted file mode 100644 index 0436fda..0000000 --- a/.github/workflows/stable.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: stable -on: - push: - tags: - - '[0-9]+.[0-9]+.[0-9]+' - -jobs: - build: - name: stable build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Use python 3.x - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: build stable release - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_DDA }} - run: | - pyb -P version=${{ github.ref }} publish upload - - - name: Create GH Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - draft: false - prerelease: false - \ No newline at end of file diff --git a/.github/workflows/unstable.yml b/.github/workflows/unstable.yml deleted file mode 100644 index d7fd8b4..0000000 --- a/.github/workflows/unstable.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: unstable -on: - push: - tags: - - '![0-9]+.[0-9]+.[0-9]+' - -jobs: - build: - name: unstable - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Use python 3.x - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: build unstable release - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_DDA }} - run: | - pyb publish upload diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6570fa..e5cbeae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,13 +4,14 @@ stages: - image .py: &py - image: "domaindrivenarchitecture/ddadevops-python:4.1.0" + image: "domaindrivenarchitecture/ddadevops-python:4.7.0" before_script: + - export RELEASE_ARTIFACT_TOKEN=$MEISSA_REPO_BUERO_RW - python --version - pip install -r requirements.txt .img: &img - image: "domaindrivenarchitecture/ddadevops-dind:4.1.0" + image: "domaindrivenarchitecture/ddadevops-dind:4.7.0" services: - docker:dind before_script: @@ -43,7 +44,7 @@ pypi-stable: <<: *tag_only stage: upload script: - - pyb -P version=$CI_COMMIT_TAG publish upload + - pyb -P version=$CI_COMMIT_TAG publish upload publish_artifacts clj-cljs-image-publish: <<: *img @@ -59,7 +60,6 @@ clj-image-publish: script: - cd infrastructure/clj && pyb image publish - python-image-publish: <<: *img <<: *tag_only diff --git a/README.md b/README.md index aac6210..1a3c411 100644 --- a/README.md +++ b/README.md @@ -205,8 +205,3 @@ For more details about our repository model see: https://repo.prod.meissa.de/mei Copyright © 2021 meissa GmbH Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") - -## License - -Copyright © 2023 meissa GmbH -Licensed under the [Apache License, Version 2.0](LICENSE) (the "License") diff --git a/build.py b/build.py index 7dfd7e2..ca56697 100644 --- a/build.py +++ b/build.py @@ -33,7 +33,7 @@ default_task = "dev" name = "ddadevops" MODULE = "not-used" PROJECT_ROOT_PATH = "." -version = "4.3.2-dev" +version = "4.7.5-dev" summary = "tools to support builds combining gopass, terraform, dda-pallet, aws & hetzner-cloud" description = __doc__ authors = [Author("meissa GmbH", "buero@meissa-gmbh.de")] @@ -46,7 +46,7 @@ license = "Apache Software License" def initialize(project): # project.build_depends_on('mockito') # project.build_depends_on('unittest-xml-reporting') - project.build_depends_on("ddadevops>=4.0.0") + project.build_depends_on("ddadevops>=4.7.0") project.set_property("verbose", True) project.get_property("filter_resources_glob").append( @@ -104,6 +104,10 @@ def initialize(project): "infrastructure/clj/build.py", "infrastructure/kotlin/build.py", ], + "release_artifacts": [], + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "dda-devops-build", } build = ReleaseMixin(project, input) @@ -179,6 +183,12 @@ def tag(project): build.tag_bump_and_push_release() +@task +def publish_artifacts(project): + build = get_devops_build(project) + build.publish_artifacts() + + def release(project): prepare(project) tag(project) diff --git a/doc/DevopsBuild.md b/doc/DevopsBuild.md index 9056dbd..0e81948 100644 --- a/doc/DevopsBuild.md +++ b/doc/DevopsBuild.md @@ -23,12 +23,9 @@ classDiagram | build_dir_name | name of dir, build is executed in | target | | build_types | list of special builds used. Valid values are ["IMAGE", "C4K", "K3S", "TERRAFORM"] | [] | | mixin_types | mixins are orthoganl to builds and represent additional capabilities. Valid Values are ["RELEASE"] | [] | -| module | module name - may result in a hierarchy like name/module | | -| name | dedicated name of the build | module | -| project_root_path | relative path to projects root. Is used to locate the target dir | | -| stage | sth. like test, int, acc or prod | | ## Example Usage + ### build.py ```python diff --git a/doc/DevopsImageBuild.md b/doc/DevopsImageBuild.md index d260631..1eb38b1 100644 --- a/doc/DevopsImageBuild.md +++ b/doc/DevopsImageBuild.md @@ -30,7 +30,7 @@ classDiagram | image_dockerhub_user | user to access docker-hub | IMAGE_DOCKERHUB_USER from env or credentials from gopass | | image_dockerhub_password | password to access docker-hub | IMAGE_DOCKERHUB_PASSWORD from env or credentials from gopass | | image_tag | tag for publishing the image | IMAGE_TAG from env | - +| image_naming | Strategy for calculate the image name. Posible values are [NAME_ONLY,NAME_AND_MODULE] |NAME_ONLY | ### Credentials Mapping defaults diff --git a/doc/ReleaseMixin.md b/doc/ReleaseMixin.md index aa4e419..6b4bbb6 100644 --- a/doc/ReleaseMixin.md +++ b/doc/ReleaseMixin.md @@ -1,5 +1,15 @@ # ReleaseMixin +- [ReleaseMixin](#releasemixin) + - [Input](#input) + - [Example Usage just for creating releases](#example-usage-just-for-creating-releases) + - [build.py](#buildpy) + - [call the build for creating a major release](#call-the-build-for-creating-a-major-release) + - [Example Usage for creating a release on forgejo / gitea \& upload the generated artifacts](#example-usage-for-creating-a-release-on-forgejo--gitea--upload-the-generated-artifacts) + - [build.py](#buildpy-1) + - [call the build](#call-the-build) + + Support for releases following the trunk-based-release flow (see https://trunkbaseddevelopment.com/) ```mermaid @@ -8,6 +18,7 @@ classDiagram prepare_release() - adjust all build files to carry the correct version & commit locally tag_and_push_release() - tag the git repo and push changes to origin update_release_type (release_type) - change the release type during run time + publish_artifacts() - publish release & artifacts to forgejo/gitea } ``` @@ -20,8 +31,12 @@ classDiagram | release_main_branch | the name of your trank | "main" | | release_primary_build_file | path to the build file having the leading version info (read & write). Valid extensions are .clj, .json, .gradle, .py | "./project.clj" | | release_secondary_build_files | list of secondary build files, version is written in. | [] | +| release_artifact_server_url | Optional: The base url of your forgejo/gitea instance to publish a release tode | | +| release_organisation | Optional: The repository organisation name | | +| release_repository_name | Optional: The repository name name | | +| release_artifacts | Optional: The list of artifacts to publish to the release generated name | [] | -## Example Usage +## Example Usage just for creating releases ### build.py @@ -36,7 +51,7 @@ PROJECT_ROOT_PATH = '..' @init def initialize(project): - project.build_depends_on("ddadevops>=4.0.0") + project.build_depends_on("ddadevops>=4.7.0") input = { "name": name, @@ -48,35 +63,108 @@ def initialize(project): "release_type": "MINOR", "release_primary_build_file": "project.clj", "release_secondary_build_files": ["package.json"], - } - - roject.build_depends_on("ddadevops>=4.0.0-dev") - + } build = ReleaseMixin(project, input) build.initialize_build_dir() @task -def prepare_release(project): - build = get_devops_build(project) - build.prepare_release() +def patch(project): + linttest(project, "PATCH") + release(project) + @task -def build(project): - print("do the build") +def minor(project): + linttest(project, "MINOR") + release(project) + @task -def publish(project): - print("publish your artefacts") +def major(project): + linttest(project, "MAJOR") + release(project) + + +@task +def dev(project): + linttest(project, "NONE") + + +@task +def prepare(project): + build = get_devops_build(project) + build.prepare_release() + @task -def after_publish(project): +def tag(project): build = get_devops_build(project) build.tag_bump_and_push_release() + + +def release(project): + prepare(project) + tag(project) + + +def linttest(project, release_type): + build = get_devops_build(project) + build.update_release_type(release_type) + #test(project) + #lint(project) +``` + +### call the build for creating a major release + +```bash +pyb major +``` + +## Example Usage for creating a release on forgejo / gitea & upload the generated artifacts + +### build.py + +```python +rom os import environ +from pybuilder.core import task, init +from ddadevops import * + +name = 'my-project' +MODULE = 'my-module' +PROJECT_ROOT_PATH = '..' + +@init +def initialize(project): + project.build_depends_on("ddadevops>=4.7.0") + + input = { + "name": name, + "module": MODULE, + "stage": "notused", + "project_root_path": PROJECT_ROOT_PATH, + "build_types": [], + "mixin_types": ["RELEASE"], + "release_type": "MINOR", + "release_primary_build_file": "project.clj", + "release_secondary_build_files": ["package.json"], + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "dda-devops-build", + "release_artifacts": ["target/doc.zip"], + } + build = ReleaseMixin(project, input) + build.initialize_build_dir() + +@task +def publish_artifacts(project): + build = get_devops_build(project) + build.publish_artifacts() ``` ### call the build ```bash -pyb prepare_release build publish after_publish +git checkout "4.7.0" +pyb publish_artifacts ``` diff --git a/doc/architecture/Domain.md b/doc/architecture/Domain.md index afb8378..3ba7c0b 100644 --- a/doc/architecture/Domain.md +++ b/doc/architecture/Domain.md @@ -88,6 +88,15 @@ classDiagram release_type release_main_branch release_current_branch + release_artifact_server_url + release_organisation + release_repository_name + release_artifact_token + } + class Artifact { + path_str + path() + type() } class Credentials { <> @@ -130,6 +139,7 @@ classDiagram TerraformDomain *-- "0..1" ProviderAws: providers Release o-- "0..1" BuildFile: primary_build_file Release o-- "0..n" BuildFile: secondary_build_files + Release "1" *-- "0..n" Artifact: release_artifacts Release "1" *-- "1" Version: version BuildFile *-- "1" Version: version C4k *-- DnsRecord: dns_record diff --git a/infrastructure/clj-cljs/build.py b/infrastructure/clj-cljs/build.py index 3b07801..6d06890 100644 --- a/infrastructure/clj-cljs/build.py +++ b/infrastructure/clj-cljs/build.py @@ -6,7 +6,7 @@ from ddadevops import * name = "ddadevops" MODULE = "clj-cljs" PROJECT_ROOT_PATH = "../.." -version = "4.3.2-dev" +version = "4.7.5-dev" @init def initialize(project): diff --git a/infrastructure/clj-cljs/image/resources/install.sh b/infrastructure/clj-cljs/image/resources/install.sh index aab2448..1167e3a 100755 --- a/infrastructure/clj-cljs/image/resources/install.sh +++ b/infrastructure/clj-cljs/image/resources/install.sh @@ -23,7 +23,7 @@ function main() { #install pyb apt -qqy install python3 python3-pip git; - pip3 install pybuilder 'ddadevops>=4.2.0' deprecation dda-python-terraform boto3 pyyaml inflection --break-system-packages; + pip3 install pybuilder 'ddadevops>=4.7.0' deprecation dda-python-terraform boto3 pyyaml inflection --break-system-packages; cleanupDocker } diff --git a/infrastructure/clj/build.py b/infrastructure/clj/build.py index d0fd84d..6093263 100644 --- a/infrastructure/clj/build.py +++ b/infrastructure/clj/build.py @@ -6,7 +6,7 @@ from ddadevops import * name = "ddadevops" MODULE = "clj" PROJECT_ROOT_PATH = "../.." -version = "4.3.2-dev" +version = "4.7.5-dev" @init def initialize(project): diff --git a/infrastructure/clj/image/resources/install.sh b/infrastructure/clj/image/resources/install.sh index 987cf0a..6694783 100755 --- a/infrastructure/clj/image/resources/install.sh +++ b/infrastructure/clj/image/resources/install.sh @@ -29,7 +29,7 @@ function main() { #install pyb apt -qqy install python3 python3-pip; - pip3 install pybuilder 'ddadevops>=4.2.0' deprecation dda-python-terraform boto3 pyyaml inflection --break-system-packages; + pip3 install pybuilder 'ddadevops>=4.7.0' deprecation dda-python-terraform boto3 pyyaml inflection --break-system-packages; cleanupDocker } diff --git a/infrastructure/ddadevops/build.py b/infrastructure/ddadevops/build.py index 302af91..da14d36 100644 --- a/infrastructure/ddadevops/build.py +++ b/infrastructure/ddadevops/build.py @@ -6,7 +6,7 @@ from ddadevops import * name = "ddadevops" MODULE = "ddadevops" PROJECT_ROOT_PATH = "../.." -version = "4.3.2-dev" +version = "4.7.5-dev" @init diff --git a/infrastructure/ddadevops/image/Dockerfile b/infrastructure/ddadevops/image/Dockerfile index 833fa70..3ec73d0 100644 --- a/infrastructure/ddadevops/image/Dockerfile +++ b/infrastructure/ddadevops/image/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.10-alpine RUN set -eux; -RUN apk add --no-cache python3 py3-pip openssl-dev bash git; +RUN apk add --no-cache python3 py3-pip openssl-dev bash git curl; RUN python3 -m pip install -U pip; RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection; diff --git a/infrastructure/dind/build.py b/infrastructure/dind/build.py index 2227a5c..aac3c14 100644 --- a/infrastructure/dind/build.py +++ b/infrastructure/dind/build.py @@ -6,7 +6,7 @@ from ddadevops import * name = "ddadevops" MODULE = "dind" PROJECT_ROOT_PATH = "../.." -version = "4.3.2-dev" +version = "4.7.5-dev" @init diff --git a/infrastructure/python/build.py b/infrastructure/python/build.py index 8a931ac..fd453d4 100644 --- a/infrastructure/python/build.py +++ b/infrastructure/python/build.py @@ -6,7 +6,7 @@ from ddadevops import * name = "ddadevops" MODULE = "python" PROJECT_ROOT_PATH = "../.." -version = "4.3.2-dev" +version = "4.7.5-dev" @init diff --git a/infrastructure/python/image/Dockerfile b/infrastructure/python/image/Dockerfile index 3d49141..bae55e0 100644 --- a/infrastructure/python/image/Dockerfile +++ b/infrastructure/python/image/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10-alpine RUN set -eux; -RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptools py3-wheel libffi-dev openssl-dev cargo bash git; +RUN apk add --no-cache build-base rust python3 python3-dev py3-pip py3-setuptools py3-wheel libffi-dev openssl-dev cargo bash git curl; RUN python3 -m pip install -U pip; RUN pip3 install pybuilder ddadevops deprecation dda-python-terraform boto3 pyyaml inflection; RUN pip3 install coverage flake8 flake8-polyfill mypy mypy-extensions pycodestyle pyflakes pylint pytest pytest-cov pytest-datafiles types-setuptools types-PyYAML; diff --git a/src/main/python/ddadevops/application/release_mixin_services.py b/src/main/python/ddadevops/application/release_mixin_services.py index c9ab06e..6c96c07 100644 --- a/src/main/python/ddadevops/application/release_mixin_services.py +++ b/src/main/python/ddadevops/application/release_mixin_services.py @@ -1,18 +1,26 @@ +import json from typing import List from pathlib import Path -from ..infrastructure import GitApi, BuildFileRepository -from ..domain import Version, Release, ReleaseType +from ..infrastructure import GitApi, ArtifactDeploymentApi, BuildFileRepository +from ..domain import Version, Release, ReleaseType, Artifact class ReleaseService: - def __init__(self, git_api: GitApi, build_file_repository: BuildFileRepository): + def __init__( + self, + git_api: GitApi, + artifact_deployment_api: ArtifactDeploymentApi, + build_file_repository: BuildFileRepository, + ): self.git_api = git_api + self.artifact_deployment_api = artifact_deployment_api self.build_file_repository = build_file_repository @classmethod def prod(cls, base_dir: str): return cls( GitApi(), + ArtifactDeploymentApi(), BuildFileRepository(base_dir), ) @@ -53,6 +61,41 @@ class ReleaseService: ) self.git_api.push_follow_tags() + def publish_artifacts(self, release: Release): + token = str(release.release_artifact_token) + release_id = self.__parse_forgejo_release_id__( + self.artifact_deployment_api.create_forgejo_release( + release.forgejo_release_api_endpoint(), + release.version.to_string(), + token, + ) + ) + + artifacts_sums = [] + for artifact in release.release_artifacts: + sha256 = self.artifact_deployment_api.calculate_sha256(artifact.path()) + sha512 = self.artifact_deployment_api.calculate_sha512(artifact.path()) + artifacts_sums += [Artifact(sha256), Artifact(sha512)] + + artifacts = release.release_artifacts + artifacts_sums + print(artifacts) + for artifact in artifacts: + print(str) + self.artifact_deployment_api.add_asset_to_release( + release.forgejo_release_asset_api_endpoint(release_id), + artifact.path(), + artifact.type(), + token, + ) + + def __parse_forgejo_release_id__(self, release_response: str) -> int: + parsed = json.loads(release_response) + try: + result = parsed["id"] + except: + raise RuntimeError(str(parsed)) + return result + def __set_version_and_commit__( self, version: Version, build_file_ids: List[str], message: str ): diff --git a/src/main/python/ddadevops/artifact_deployment_mixin.py b/src/main/python/ddadevops/artifact_deployment_mixin.py deleted file mode 100644 index 144fe53..0000000 --- a/src/main/python/ddadevops/artifact_deployment_mixin.py +++ /dev/null @@ -1,56 +0,0 @@ -from pybuilder.core import Project -from .devops_build import DevopsBuild - -# """ -# Functional Req: - -# General process for deploying prebuilt (meissa) binaries to our own repo server. - -# [-1] -# Building is handled by other entities -# is another pybuilder task -# the binary is reachable with devops.build_path() -# we might need to establish a "build" that does lein builds for us -# we might need to establish a "build" that does gradlew build for us -# same for all other projects that produce binaries -# currently the c4k_build.py just creates the auth and config yamls - -# [0] -# get artifact deployment url -# Base url: https://repo.prod.meissa.de/api/v1/repos/ -# Changeable: /meissa/provs/ -# persitent suffix to url: releases -# name is accessible from input - -# [1] -# get release token -# could be an api token for repo.prod.meissa.de -# credential mapping as described in the docs - -# [2] -# get release tag -# is the version of the project -# get from gitApi - -# [3] -# post a json message containting [2] to [0], watching stdout for answers -# authorized by [1] -# validate if [3] was successful by reading stdout -# or create error message containing ID of release - -# [4] -# get release-id from stdout of [3] -# print release-id - -# [5] -# generate sha256 sums & generate sha512 sums of results of [-1] - -# [6] -# push results of [-1] & [5] to [0]/[4] - -# """ - - -class ArtifactDeploymentMixin(DevopsBuild): - def __init__(self, project: Project, inp: dict): - super().__init__(project, inp) diff --git a/src/main/python/ddadevops/domain/__init__.py b/src/main/python/ddadevops/domain/__init__.py index 234c48b..6bff258 100644 --- a/src/main/python/ddadevops/domain/__init__.py +++ b/src/main/python/ddadevops/domain/__init__.py @@ -17,6 +17,7 @@ from .provider_hetzner import Hetzner from .provider_aws import Aws from .provs_k3s import K3s from .release import Release +from .artifact import Artifact from .credentials import Credentials, CredentialMapping, GopassType from .version import Version from .build_file import BuildFileType, BuildFile diff --git a/src/main/python/ddadevops/domain/artifact.py b/src/main/python/ddadevops/domain/artifact.py new file mode 100644 index 0000000..f7ae720 --- /dev/null +++ b/src/main/python/ddadevops/domain/artifact.py @@ -0,0 +1,46 @@ +from enum import Enum +from pathlib import Path +from .common import ( + Validateable, +) + + +class ArtifactType(Enum): + TEXT = 0 + JAR = 1 + + +class Artifact(Validateable): + def __init__(self, path: str): + self.path_str = path + + def path(self) -> Path: + return Path(self.path_str) + + def type(self) -> str: + suffix = self.path().suffix + match suffix: + case ".jar": + return "application/x-java-archive" + case ".js": + return "application/x-javascript" + case _: + return "text/plain" + + def validate(self): + result = [] + result += self.__validate_is_not_empty__("path_str") + try: + Path(self.path_str) + except Exception as e: + result += [f"path was not a valid: {e}"] + return result + + def __str__(self): + return str(self.path()) + + def __eq__(self, other): + return other and self.__str__() == other.__str__() + + def __hash__(self) -> int: + return self.__str__().__hash__() diff --git a/src/main/python/ddadevops/domain/init_service.py b/src/main/python/ddadevops/domain/init_service.py index 2d5ab56..8529665 100644 --- a/src/main/python/ddadevops/domain/init_service.py +++ b/src/main/python/ddadevops/domain/init_service.py @@ -8,7 +8,7 @@ from .provider_digitalocean import Digitalocean from .provider_hetzner import Hetzner from .c4k import C4k from .image import Image -from .release import ReleaseType +from .release import ReleaseType, Release from ..infrastructure import BuildFileRepository, CredentialsApi, EnvironmentApi, GitApi @@ -69,6 +69,7 @@ class InitService: Path(primary_build_file_id) ) version = primary_build_file.get_version() + default_mappings += Release.get_mapping_default() credentials = Credentials(inp, default_mappings) authorization = self.authorization(credentials) @@ -111,9 +112,8 @@ class InitService: result = {} for name in credentials.mappings.keys(): mapping = credentials.mappings[name] - env_value = self.environment_api.get(mapping.name_for_environment()) - if env_value: - result[name] = env_value + if self.environment_api.is_defined(mapping.name_for_environment()): + result[name] = self.environment_api.get(mapping.name_for_environment()) else: if mapping.gopass_type() == GopassType.FIELD: result[name] = self.credentials_api.gopass_field_from_path( diff --git a/src/main/python/ddadevops/domain/release.py b/src/main/python/ddadevops/domain/release.py index e8a6db5..490db9b 100644 --- a/src/main/python/ddadevops/domain/release.py +++ b/src/main/python/ddadevops/domain/release.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import Optional, List, Dict from pathlib import Path from .common import ( Validateable, @@ -7,6 +7,9 @@ from .common import ( from .version import ( Version, ) +from .artifact import ( + Artifact, +) class Release(Validateable): @@ -21,6 +24,13 @@ class Release(Validateable): "release_secondary_build_files", [] ) self.version = version + self.release_artifact_server_url = inp.get("release_artifact_server_url") + self.release_organisation = inp.get("release_organisation") + self.release_repository_name = inp.get("release_repository_name") + self.release_artifact_token = inp.get("release_artifact_token") + self.release_artifacts = [] + for a in inp.get("release_artifacts", []): + self.release_artifacts.append(Artifact(a)) def update_release_type(self, release_type: ReleaseType): self.release_type = release_type @@ -53,10 +63,44 @@ class Release(Validateable): and self.release_type != ReleaseType.NONE and self.release_main_branch != self.release_current_branch ): - result.append(f"Releases are allowed only on {self.release_main_branch}") + result.append( + f"Releases are allowed only on {self.release_main_branch}" + ) + return result + + def validate_for_artifact(self): + result = [] + result += self.__validate_is_not_empty__("release_artifact_server_url") + result += self.__validate_is_not_empty__("release_organisation") + result += self.__validate_is_not_empty__("release_repository_name") + result += self.__validate_is_not_empty__("release_artifact_token") return result def build_files(self) -> List[str]: result = [self.release_primary_build_file] result += self.release_secondary_build_files return result + + def forgejo_release_api_endpoint(self) -> str: + validation = self.validate_for_artifact() + if validation != []: + raise RuntimeError(f"not valid for creating artifacts: {validation}") + + server_url = self.release_artifact_server_url.removeprefix("/").removesuffix( + "/" + ) + organisation = self.release_organisation.removeprefix("/").removesuffix("/") + repository = self.release_repository_name.removeprefix("/").removesuffix("/") + return f"{server_url}/api/v1/repos/{organisation}/{repository}/releases" + + def forgejo_release_asset_api_endpoint(self, release_id: int) -> str: + return f"{self.forgejo_release_api_endpoint()}/{release_id}/assets" + + @classmethod + def get_mapping_default(cls) -> List[Dict[str, str]]: + return [ + { + "gopass_path": "server/meissa/repo/buero-rw", + "name": "release_artifact_token", + } + ] diff --git a/src/main/python/ddadevops/domain/version.py b/src/main/python/ddadevops/domain/version.py index 2f2096a..3c9ac0e 100644 --- a/src/main/python/ddadevops/domain/version.py +++ b/src/main/python/ddadevops/domain/version.py @@ -32,12 +32,6 @@ class Version(Validateable): self.snapshot_suffix = snapshot_suffix self.default_snapshot_suffix = default_snapshot_suffix - def __eq__(self, other): - return other and self.to_string() == other.to_string() - - def __hash__(self) -> int: - return self.to_string().__hash__() - def is_snapshot(self): return self.snapshot_suffix is not None @@ -139,3 +133,9 @@ class Version(Validateable): snapshot_suffix=None, version_str=None, ) + + def __eq__(self, other): + return other and self.to_string() == other.to_string() + + def __hash__(self) -> int: + return self.to_string().__hash__() diff --git a/src/main/python/ddadevops/infrastructure/__init__.py b/src/main/python/ddadevops/infrastructure/__init__.py index 1bb11b5..1a520ed 100644 --- a/src/main/python/ddadevops/infrastructure/__init__.py +++ b/src/main/python/ddadevops/infrastructure/__init__.py @@ -7,5 +7,6 @@ from .infrastructure import ( CredentialsApi, GitApi, TerraformApi, + ArtifactDeploymentApi, ) from .repository import DevopsRepository, BuildFileRepository diff --git a/src/main/python/ddadevops/infrastructure/infrastructure.py b/src/main/python/ddadevops/infrastructure/infrastructure.py index 99ec45c..4a65c63 100644 --- a/src/main/python/ddadevops/infrastructure/infrastructure.py +++ b/src/main/python/ddadevops/infrastructure/infrastructure.py @@ -58,23 +58,21 @@ class ImageApi: ) def drun(self, name: str): - self.execution_api.execute_live( - f'docker run -it --entrypoint="" {name} /bin/bash' + run( + f'docker run -it --entrypoint="" {name} /bin/bash', + shell=True, + check=True, ) def dockerhub_login(self, username: str, password: str): self.execution_api.execute_secure( f"docker login --username {username} --password {password}", - "docker login --username ***** --password *****" + "docker login --username ***** --password *****", ) def dockerhub_publish(self, name: str, username: str, tag: str): - self.execution_api.execute_live( - f"docker tag {name} {username}/{name}:{tag}" - ) - self.execution_api.execute_live( - f"docker push {username}/{name}:{tag}" - ) + self.execution_api.execute_live(f"docker tag {name} {username}/{name}:{tag}") + self.execution_api.execute_live(f"docker push {username}/{name}:{tag}") def test(self, name: str, path: Path): self.execution_api.execute_live( @@ -95,14 +93,24 @@ class ExecutionApi: check=check, stdout=PIPE, stderr=PIPE, - text=True).stdout + text=True, + ).stdout output = output.rstrip() except CalledProcessError as exc: - print(f"Command failed with code: {exc.returncode} and message: {exc.stderr}") + print( + f"Command failed with code: {exc.returncode} and message: {exc.stderr}" + ) raise exc return output - def execute_secure(self, command: str, sanitized_command: str, dry_run=False, shell=True, check=True): + def execute_secure( + self, + command: str, + sanitized_command: str, + dry_run=False, + shell=True, + check=True, + ): try: output = self.execute(command, dry_run, shell, check) return output @@ -128,6 +136,9 @@ class EnvironmentApi: def get(self, key): return environ.get(key) + def is_defined(self, key): + return key in environ + class CredentialsApi: def __init__(self): @@ -206,3 +217,53 @@ class GitApi: class TerraformApi: pass + + +class ArtifactDeploymentApi: + def __init__(self): + self.execution_api = ExecutionApi() + + def create_forgejo_release(self, api_endpoint_url: str, tag: str, token: str): + command = ( + f'curl -X "POST" "{api_endpoint_url}" ' + + ' -H "accept: application/json" -H "Content-Type: application/json"' + + f' -d \'{{ "body": "Provides files for release {tag}", "tag_name": "{tag}"}}\'' + ) # noqa: E501 + print(command + ' -H "Authorization: token xxxx"') + return self.execution_api.execute_secure( + command=command + f' -H "Authorization: token {token}"', + sanitized_command=command + ' -H "Authorization: token xxxx"', + ) + + def add_asset_to_release( + self, + api_endpoint_url: str, + attachment: Path, + attachment_type: str, + token: str, + ): + command = ( + f'curl -X "POST" "{api_endpoint_url}"' + + ' -H "accept: application/json"' + + ' -H "Content-Type: multipart/form-data"' + + f' -F "attachment=@{attachment};type={attachment_type}"' + ) # noqa: E501 + print(command + ' -H "Authorization: token xxxx"') + return self.execution_api.execute_secure( + command=command + f' -H "Authorization: token {token}"', + sanitized_command=command + ' -H "Authorization: token xxxx"', + ) + + def calculate_sha256(self, path: Path): + shasum = f"{path}.sha256" + self.execution_api.execute( + f"sha256sum {path} > {shasum}", + ) + return shasum + + def calculate_sha512(self, path: Path): + shasum = f"{path}.sha512" + self.execution_api.execute( + f"sha512sum {path} > {shasum}", + ) + return shasum diff --git a/src/main/python/ddadevops/release_mixin.py b/src/main/python/ddadevops/release_mixin.py index 3d7defd..5ece524 100644 --- a/src/main/python/ddadevops/release_mixin.py +++ b/src/main/python/ddadevops/release_mixin.py @@ -26,3 +26,8 @@ class ReleaseMixin(DevopsBuild): devops = self.devops_repo.get_devops(self.project) release = devops.mixins[MixinType.RELEASE] self.release_service.tag_bump_and_push_release(release) + + def publish_artifacts(self): + devops = self.devops_repo.get_devops(self.project) + release = devops.mixins[MixinType.RELEASE] + self.release_service.publish_artifacts(release) diff --git a/src/test/python/application/test_release_mixin_services.py b/src/test/python/application/test_release_mixin_services.py index 40f6553..976d7c5 100644 --- a/src/test/python/application/test_release_mixin_services.py +++ b/src/test/python/application/test_release_mixin_services.py @@ -1,18 +1,22 @@ import pytest from pathlib import Path from src.main.python.ddadevops.domain import ( - ReleaseType, + ReleaseType, MixinType, ) from src.test.python.domain.helper import ( BuildFileRepositoryMock, GitApiMock, + ArtifactDeploymentApiMock, build_devops, ) from src.main.python.ddadevops.application import ReleaseService + def test_sould_update_release_type(): - sut = ReleaseService(GitApiMock(), BuildFileRepositoryMock("build.py")) + sut = ReleaseService( + GitApiMock(), ArtifactDeploymentApiMock(), BuildFileRepositoryMock("build.py") + ) devops = build_devops({}) release = devops.mixins[MixinType.RELEASE] sut.update_release_type(release, "MAJOR") @@ -20,3 +24,40 @@ def test_sould_update_release_type(): with pytest.raises(Exception): sut.update_release_type(release, "NOT_EXISTING") + + +def test_sould_publish_artifacts(): + mock = ArtifactDeploymentApiMock(release='{"id": 2345}') + sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock()) + devops = build_devops( + { + "release_artifacts": ["target/art"], + "release_artifact_server_url": "http://repo.test/", + "release_organisation": "orga", + "release_repository_name": "repo", + } + ) + release = devops.mixins[MixinType.RELEASE] + sut.publish_artifacts(release) + assert "http://repo.test/api/v1/repos/orga/repo/releases/2345/assets" == mock.add_asset_to_release_api_endpoint + +def test_sould_throw_exception_if_there_was_an_error_in_publish_artifacts(): + devops = build_devops( + { + "release_artifacts": ["target/art"], + "release_artifact_server_url": "http://repo.test/", + "release_organisation": "orga", + "release_repository_name": "repo", + } + ) + release = devops.mixins[MixinType.RELEASE] + + with pytest.raises(Exception): + mock = ArtifactDeploymentApiMock(release='') + sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock()) + sut.publish_artifacts(release) + + with pytest.raises(Exception): + mock = ArtifactDeploymentApiMock(release='{"message": "there was an error", "url":"some-url"}') + sut = ReleaseService(GitApiMock(), mock, BuildFileRepositoryMock()) + sut.publish_artifacts(release) diff --git a/src/test/python/domain/helper.py b/src/test/python/domain/helper.py index 05161d0..7c6df4c 100644 --- a/src/test/python/domain/helper.py +++ b/src/test/python/domain/helper.py @@ -53,6 +53,11 @@ def devops_config(overrides: dict) -> dict: "release_current_branch": "my_feature", "release_primary_build_file": "./package.json", "release_secondary_build_file": [], + "release_artifacts": [], + "release_artifact_token": "release_artifact_token", + "release_artifact_server_url": None, + "release_organisation": None, + "release_repository_name": None, "credentials_mappings": [ { "gopass_path": "a/path", @@ -99,6 +104,9 @@ class EnvironmentApiMock: def get(self, key): return self.mappings.get(key, None) + def is_defined(self, key): + return key in self.mappings + class CredentialsApiMock: def __init__(self, mappings): @@ -150,3 +158,28 @@ class GitApiMock: def checkout(self, branch: str): pass + + +class ArtifactDeploymentApiMock: + def __init__(self, release=""): + self.release = release + self.create_forgejo_release_count = 0 + self.add_asset_to_release_count = 0 + self.add_asset_to_release_api_endpoint = "" + + def create_forgejo_release(self, api_endpoint: str, tag: str, token: str): + self.create_forgejo_release_count += 1 + return self.release + + def add_asset_to_release( + self, api_endpoint: str, attachment: str, attachment_type: str, token: str + ): + self.add_asset_to_release_api_endpoint = api_endpoint + self.add_asset_to_release_count += 1 + pass + + def calculate_sha256(self, path: Path): + return f"{path}.sha256" + + def calculate_sha512(self, path: Path): + return f"{path}.sha512" diff --git a/src/test/python/domain/test_artifact.py b/src/test/python/domain/test_artifact.py new file mode 100644 index 0000000..86e5324 --- /dev/null +++ b/src/test/python/domain/test_artifact.py @@ -0,0 +1,32 @@ +import pytest +from pybuilder.core import Project +from pathlib import Path +from src.main.python.ddadevops.domain import ( + Validateable, + DnsRecord, + Devops, + BuildType, + MixinType, + Artifact, + Image, +) +from .helper import build_devops, devops_config + + +def test_sould_validate_release(): + sut = Artifact("x") + assert sut.is_valid() + + sut = Artifact(None) + assert not sut.is_valid() + +def test_should_calculate_type(): + sut = Artifact("x.jar") + assert "application/x-java-archive" == sut.type() + + sut = Artifact("x.js") + assert "application/x-javascript" == sut.type() + + sut = Artifact("x.jar.sha256") + assert "text/plain" == sut.type() + diff --git a/src/test/python/domain/test_devops_factory.py b/src/test/python/domain/test_devops_factory.py index b9f4008..26cc319 100644 --- a/src/test/python/domain/test_devops_factory.py +++ b/src/test/python/domain/test_devops_factory.py @@ -4,6 +4,7 @@ from src.main.python.ddadevops.domain import ( Version, BuildType, MixinType, + Artifact, ) @@ -50,6 +51,7 @@ def test_devops_creation(): assert sut is not None assert sut.specialized_builds[BuildType.C4K] is not None +def test_release_devops_creation(): sut = DevopsFactory().build_devops( { "stage": "test", @@ -67,6 +69,30 @@ def test_devops_creation(): assert sut is not None assert sut.mixins[MixinType.RELEASE] is not None + sut = DevopsFactory().build_devops( + { + "stage": "test", + "name": "mybuild", + "module": "test_image", + "project_root_path": "../../..", + "build_types": [], + "mixin_types": ["RELEASE"], + "release_main_branch": "main", + "release_current_branch": "my_feature", + "release_config_file": "project.clj", + "release_artifacts": ["x.jar"], + "release_artifact_token": "y", + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "provs", + }, + Version.from_str("1.0.0", "SNAPSHOT"), + ) + + release = sut.mixins[MixinType.RELEASE] + assert release is not None + assert Artifact("x.jar") == release.release_artifacts[0] + def test_on_merge_input_should_win(): sut = DevopsFactory() diff --git a/src/test/python/domain/test_release.py b/src/test/python/domain/test_release.py index c23bc39..968ce69 100644 --- a/src/test/python/domain/test_release.py +++ b/src/test/python/domain/test_release.py @@ -1,3 +1,4 @@ +import pytest from pybuilder.core import Project from pathlib import Path from src.main.python.ddadevops.domain import ( @@ -61,3 +62,74 @@ def test_sould_calculate_build_files(): Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), ) assert ["project.clj", "package.json"] == sut.build_files() + + +def test_should_calculate_forgejo_release_api_endpoint(): + sut = Release( + devops_config( + { + "release_artifacts": [], + "release_artifact_token": "y", + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + assert ( + "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases" + == sut.forgejo_release_api_endpoint() + ) + + sut = Release( + devops_config( + { + "release_artifacts": ["x"], + "release_artifact_token": "y", + "release_artifact_server_url": "https://repo.prod.meissa.de/", + "release_organisation": "/meissa/", + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + assert ( + "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases" + == sut.forgejo_release_api_endpoint() + ) + assert( + "/meissa/" + == sut.release_organisation + ) + + with pytest.raises(Exception): + sut = Release( + devops_config( + { + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": None, + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + sut.forgejo_release_api_endpoint() + +def test_should_calculate_forgejo_release_asset_api_endpoint(): + sut = Release( + devops_config( + { + "release_artifacts": ["x"], + "release_artifact_token": "y", + "release_artifact_server_url": "https://repo.prod.meissa.de", + "release_organisation": "meissa", + "release_repository_name": "provs", + } + ), + Version.from_str("1.3.1-SNAPSHOT", "SNAPSHOT"), + ) + assert ( + "https://repo.prod.meissa.de/api/v1/repos/meissa/provs/releases/123/assets" + == sut.forgejo_release_asset_api_endpoint(123) + ) diff --git a/src/test/python/test_release_mixin.py b/src/test/python/test_release_mixin.py index 291eb73..a2539a6 100644 --- a/src/test/python/test_release_mixin.py +++ b/src/test/python/test_release_mixin.py @@ -14,6 +14,8 @@ def test_release_mixin(tmp_path): copy_resource(Path("package.json"), tmp_path) project = Project(str_tmp_path, name="name") + os.environ["RELEASE_ARTIFACT_TOKEN"] = "ratoken" + sut = ReleaseMixin( project, devops_config(