WIP
All checks were successful
run tests / check (push) Successful in 22s
run tests / release (push) Successful in 16s

This commit is contained in:
2025-12-05 21:05:59 +01:00
parent bd50bef4ad
commit 1add198c39
37 changed files with 2598 additions and 0 deletions

1
.envrc Normal file
View File

@@ -0,0 +1 @@
use_flake

1
.gitea/CODEOWNERS Normal file
View File

@@ -0,0 +1 @@
* @puzzleYOU/team-gelb

1
.gitea/release.yaml Normal file
View File

@@ -0,0 +1 @@
version_descriptor: version.txt

102
.gitea/workflows/check.yaml Normal file
View File

@@ -0,0 +1,102 @@
name: run tests
on:
- pull_request
- workflow_call
jobs:
unittest:
runs-on: action-runner
steps:
- uses: actions/checkout@v4
- name: unittest
shell: nix develop --command bash -- {0}
run: just test-python
test-declare-default:
runs-on: action-runner
steps:
- uses: actions/checkout@v4
- uses: ./declare
with:
configure_runner_environment: false
- run: set -u; echo "$RELEASE_PROJECT_CURRENT_VERSION"
test-declare-with-release-yaml:
runs-on: action-runner
steps:
- uses: actions/checkout@v4
# skip login locally
- if: github.repository == 'actions/release'
uses: https://gitea.puzzleyou.net/actions/configure-runner-environment@master
- uses: ./release
with:
dry_run: true
configure_runner_environment: false
- run: set -u; echo "$RELEASE_PROJECT_CURRENT_VERSION"
test-declare-directly:
runs-on: action-runner
steps:
- uses: actions/checkout@v4
- uses: ./declare
with:
configure_runner_environment: false
version_descriptor: test-assets/version.txt
repository: actions/release
artefact_type: oci_image
artefact_name: default-image
deployment_type: helm_release
deployment_release_name: example-testing
- uses: ./add-artefact
with:
type: "oci_image"
name: "some-docker-image"
repository: "special"
version_descriptor: "test-assets/Cargo.toml"
- uses: ./add-artefact
with:
type: tarball
filename: "test-assets/foo.tar.gz"
package_name: foo
- uses: ./add-artefact
with:
type: wheel
pattern: "test-assets/wheels/*.whl"
- uses: ./add-artefact
with:
type: sdist
filename: "test-assets/pypkgs.42.tar.gz"
- uses: ./add-artefact
with:
type: npm
directory: "test-assets/browser/dist"
- uses: ./add-deployment
with:
type: helm_release
release_name: other-testing
image_paths: image.foo.tag image.bar.tag
namespace: different-testing
repository: europe-docker.hetzner.cloud/puzzleyou/helm
condition: always
- name: ensure that environment variables are set
run: env | grep "RELEASE_IMAGE_LOCAL_NAME_SOME_DOCKER_IMAGE"
- name: dump release environment variables
run: env | grep "RELEASE_"
- name: dump project description
uses: ./dump

View File

@@ -0,0 +1,18 @@
name: run tests
on:
push:
branches:
- master
- testing
jobs:
check:
uses: ./.gitea/workflows/check.yaml
release:
needs: ["check"]
runs-on: action-runner
steps:
- uses: actions/checkout@v4
- uses: ./declare # TODO single action
- uses: ./release

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/.direnv/
__pycache__

60
add-artefact/action.yaml Normal file
View File

@@ -0,0 +1,60 @@
name: "add artefact to release project"
description: "add additional artefact to initialized project"
inputs:
type:
required: true
description: "known types: oci_image, tarball, wheel, sdist, npm"
repository:
required: false
description: "allowed for oci_image, tarbal, wheel, sdist"
default: ""
name:
required: false
description: "required for oci_image"
default: ""
filename:
required: false
description: "required for tarball, sdist"
default: ""
package_name:
required: false
description: "required for tarball"
default: ""
pattern:
required: false
description: "required for wheel"
default: ""
directory:
required: false
description: "required for npm"
default: ""
version_descriptor:
required: false
description: "allowed for all"
default: ""
runs:
using: composite
steps:
- name: add artefact
run: |
nix run ${{ github.action_path }} -- \
add-artefact \
--state "${RELEASE_ACTION_STATEFILE}" \
--artefact-type "${{ inputs.type }}" \
--artefact-repository "${{ inputs.repository }}" \
--artefact-name "${{ inputs.name }}" \
--artefact-filename "${{ inputs.filename }}" \
--artefact-package-name "${{ inputs.package_name }}" \
--artefact-pattern "${{ inputs.pattern }}" \
--artefact-directory "${{ inputs.directory }}" \
--version-descriptor "${{ inputs.version_descriptor }}" \
--write-env-vars-to-filename "$GITHUB_ENV"

View File

@@ -0,0 +1,50 @@
name: "add deployment to release project"
description: "add additional deployments to initialized project"
inputs:
type:
required: true
description: "known types: helm_release"
release_name:
required: true
description: "helm release name"
default: ""
condition:
required: false
description: |
when should the deployment be updated?
choices: always, never, pre_release_only, release_only
default: ""
image_paths:
required: false
description: "space separated list of paths to image tags in helm values"
default: ""
namespace:
required: false
description: "kubernetes namespace of the release"
default: ""
repository:
required: false
description: "helm chart registry"
default: ""
runs:
using: composite
steps:
- name: add deployment
run: |
nix run ${{ github.action_path }} -- \
add-deployment \
--state "${RELEASE_ACTION_STATEFILE}" \
--deployment-type "${{ inputs.type }}" \
--deployment-release-name "${{ inputs.release_name }}" \
--deployment-condition "${{ inputs.condition }}" \
--deployment-image-paths "${{ inputs.image_paths }}" \
--deployment-namespace "${{ inputs.namespace }}" \
--deployment-repository "${{ inputs.repository }}" \
--write-env-vars-to-filename "$GITHUB_ENV"

201
declare/action.yaml Normal file
View File

@@ -0,0 +1,201 @@
name: "declare project for release"
description: "start release configuration by declaring a project"
inputs:
filename:
required: false
default: ".gitea/release.yaml"
description: |
location of release.yaml file. will be ignored if version_descriptor is
set.
version_descriptor:
required: false
default: ""
configure_runner_environment:
required: false
default: true
repository:
required: false
default: "${{ github.repository }}"
gitea_instance:
required: false
default: "https://gitea.puzzleyou.net"
artefact_type:
required: false
description: "known types: oci_image, tarball, wheel, sdist, npm"
artefact_repository:
required: false
description: "allowed for oci_image, tarbal, wheel, sdist"
default: ""
artefact_name:
required: false
description: "required for oci_image"
default: ""
artefact_filename:
required: false
description: "required for tarball, sdist"
default: ""
artefact_package_name:
required: false
description: "required for tarball"
default: ""
artefact_pattern:
required: false
description: "required for wheel"
default: ""
artefact_directory:
required: false
description: "required for npm"
default: ""
artefact_version_descriptor:
required: false
description: "allowed for all"
default: ""
deployment_type:
required: false
description: "known types: helm_release"
deployment_release_name:
required: false
description: "helm release name"
default: ""
deployment_condition:
required: false
description: |
when should the deployment be updated?
choices: always, never, pre_release_only, release_only
default: ""
deployment_image_paths:
required: false
description: "space separated list of paths to image tags in helm values"
default: ""
deployment_namespace:
required: false
description: "kubernetes namespace of the release"
default: ""
deployment_repository:
required: false
description: "helm chart registry"
default: ""
runs:
using: composite
steps:
- if: inputs.configure_runner_environment == 'true'
uses: https://gitea.puzzleyou.net/actions/configure-runner-environment@master
- name: init action state
run: |
if [[ ! -z "${RELEASE_PROJECT_CURRENT_VERSION}" ]]; then
echo "already set up."
exit 0
fi
RELEASE_ACTION_STATEFILE=$(mktemp)
echo "[release] statefile: $RELEASE_ACTION_STATEFILE"
echo "RELEASE_ACTION_STATEFILE="$RELEASE_ACTION_STATEFILE"" \
>> "$GITHUB_ENV"
- name: declare release project
if: inputs.version_descriptor == ''
run: |
if [[ ! -z "${RELEASE_PROJECT_CURRENT_VERSION}" ]]; then
echo "already set up."
exit 0
fi
if [[ "${{ github.ref_name }}" == "master" || "${{ github.ref_name }}" == "main" ]]; then
IS_PRE_RELEASE="0"
else
IS_PRE_RELEASE="1"
fi
nix run ${{ github.action_path }} -- \
declare \
--state "${RELEASE_ACTION_STATEFILE}" \
--release-yaml-filename "${{ inputs.filename }}" \
--gitea-instance "${{ inputs.gitea_instance }}" \
--release-repository-name "${{ inputs.repository }}" \
--release-ref-name "${{ github.ref_name }}" \
--release-run-number "${{ github.run_number }}" \
--release-commit-sha "${{ github.sha }}" \
--is-pre-release "${IS_PRE_RELEASE}"
nix run ${{ github.action_path }} -- \
dump \
--state "${RELEASE_ACTION_STATEFILE}" \
--write-env-vars-to-filename "$GITHUB_ENV"
- name: declare release project
if: inputs.version_descriptor != ''
run: |
if [[ ! -z "${RELEASE_PROJECT_CURRENT_VERSION}" ]]; then
echo "already set up."
exit 0
fi
if [[ "${{ github.ref_name }}" == "master" || "${{ github.ref_name }}" == "main" ]]; then
IS_PRE_RELEASE="0"
else
IS_PRE_RELEASE="1"
fi
nix run ${{ github.action_path }} -- \
declare \
--state "${RELEASE_ACTION_STATEFILE}" \
--version-descriptor "${{ inputs.version_descriptor }}" \
--gitea-instance "${{ inputs.gitea_instance }}" \
--release-repository-name "${{ inputs.repository }}" \
--release-ref-name "${{ github.ref_name }}" \
--release-run-number "${{ github.run_number }}" \
--release-commit-sha "${{ github.sha }}" \
--is-pre-release "${IS_PRE_RELEASE}" \
if [[ ! -z "${{ inputs.artefact_type }}" ]]; then
nix run ${{ github.action_path }} -- \
add-artefact \
--state "${RELEASE_ACTION_STATEFILE}" \
--artefact-type "${{ inputs.artefact_type }}" \
--artefact-repository "${{ inputs.artefact_repository }}" \
--artefact-name "${{ inputs.artefact_name }}" \
--artefact-filename "${{ inputs.artefact_filename }}" \
--artefact-package-name "${{ inputs.artefact_package_name }}" \
--artefact-pattern "${{ inputs.artefact_pattern }}" \
--artefact-directory "${{ inputs.artefact_directory }}" \
--version-descriptor "${{ inputs.artefact_version_descriptor }}"
fi
if [[ ! -z "${{ inputs.deployment_type }}" ]]; then
nix run ${{ github.action_path }} -- \
add-deployment \
--state "${RELEASE_ACTION_STATEFILE}" \
--deployment-type "${{ inputs.deployment_type }}" \
--deployment-release-name "${{ inputs.deployment_release_name }}" \
--deployment-condition "${{ inputs.deployment_condition }}" \
--deployment-image-paths "${{ inputs.deployment_image_paths }}" \
--deployment-namespace "${{ inputs.deployment_namespace }}" \
--deployment-repository "${{ inputs.deployment_repository }}"
fi
nix run ${{ github.action_path }} -- \
dump \
--state "${RELEASE_ACTION_STATEFILE}" \
--write-env-vars-to-filename "$GITHUB_ENV"

11
dump/action.yaml Normal file
View File

@@ -0,0 +1,11 @@
name: "dump project description"
description: "dump current project description"
runs:
using: composite
steps:
- name: dump project description
run: |
nix run ${{ github.action_path }} -- \
dump \
--state "${RELEASE_ACTION_STATEFILE}"

61
flake.lock generated Normal file
View File

@@ -0,0 +1,61 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1764560356,
"narHash": "sha256-M5aFEFPppI4UhdOxwdmceJ9bDJC4T6C6CzCK1E2FZyo=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "6c8f0cca84510cc79e09ea99a299c9bc17d03cb6",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-25.05",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

53
flake.nix Normal file
View File

@@ -0,0 +1,53 @@
{
description = "puzzleYOU release action";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
flake-utils.url = "github:numtide/flake-utils";
};
outputs = { self, nixpkgs, flake-utils, ... }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs { inherit system; };
python = pkgs.python313.withPackages (ps: with ps; [
isort
flake8
semver
toml
requests
pyyaml
]);
pythonPackage = pkgs.python3Packages.buildPythonPackage {
name = "release-action";
src = ./.;
};
in
{
devShells.default = pkgs.mkShell {
buildInputs = [
pkgs.envsubst
pkgs.just
pkgs.gitea-actions-runner
python
];
};
packages.default = pkgs.writers.writePython3Bin
"release-action"
{
libraries = with pkgs.python3Packages; [
semver # TODO move to setup.py?
toml
requests
pythonPackage
pyyaml
];
}
(builtins.readFile ./src/main.py)
;
}
);
}

27
justfile Normal file
View File

@@ -0,0 +1,27 @@
test: test-python test-workflows
test-python:
python3 src/test.py
flake8
isort . --check
test-workflows:
act_runner exec \
--image "-self-hosted" \
--event pull_request \
--workflows ./.gitea/workflows/check.yaml \
--job test-declare-with-release-yaml
act_runner exec \
--image "-self-hosted" \
--event pull_request \
--workflows ./.gitea/workflows/check.yaml \
--job test-declare-directly
act_runner exec \
--image "-self-hosted" \
--event pull_request \
--workflows ./.gitea/workflows/check.yaml \
--job test-declare-default
# --image "europe-docker.pkg.dev/puzzle-and-play/docker/action-runner-job:latest" \

70
release/action.yaml Normal file
View File

@@ -0,0 +1,70 @@
name: "release project"
description: "publish artifacte, update deployments and create gitea release"
inputs:
dry_run:
required: false
description: "do not change external state"
default: false
configure_runner_environment:
required: false
default: true
runs:
using: composite
steps:
- name: declare project if neccessary
run: |
if [[ ! -z "${RELEASE_PROJECT_CURRENT_VERSION}" ]]; then
echo "already set up."
exit 0
fi
RELEASE_ACTION_STATEFILE=$(mktemp)
echo "[release] statefile: $RELEASE_ACTION_STATEFILE"
echo "RELEASE_ACTION_STATEFILE="$RELEASE_ACTION_STATEFILE"" \
>> "$GITHUB_ENV"
if [[ "${{ github.ref_name }}" == "master" || "${{ github.ref_name }}" == "main" ]]; then
IS_PRE_RELEASE="0"
else
IS_PRE_RELEASE="1"
fi
nix run ${{ github.action_path }} -- \
declare \
--state "${RELEASE_ACTION_STATEFILE}" \
--release-yaml-filename ".gitea/release.yaml" \
--gitea-instance "https://gitea.puzzleyou.net" \
--release-repository-name "${{ github.repository }}" \
--release-ref-name "${{ github.ref_name }}" \
--release-run-number "${{ github.run_number }}" \
--release-commit-sha "${{ github.sha }}" \
--is-pre-release "${IS_PRE_RELEASE}"
nix run ${{ github.action_path }} -- \
dump \
--state "${RELEASE_ACTION_STATEFILE}" \
--write-env-vars-to-filename "$GITHUB_ENV"
- name: publish artefacts
run: |
nix run ${{ github.action_path }} -- \
publish-artefacts \
--state "${RELEASE_ACTION_STATEFILE}" \
--dry-run "${{ inputs.dry_run }}"
- name: update deployments
run: |
nix run ${{ github.action_path }} -- \
update-deployments \
--state "${RELEASE_ACTION_STATEFILE}" \
--dry-run "${{ inputs.dry_run }}"
- name: create release
run: |
nix run ${{ github.action_path }} -- \
create-release \
--state "${RELEASE_ACTION_STATEFILE}" \
--dry-run "${{ inputs.dry_run }}"

6
setup.py Normal file
View File

@@ -0,0 +1,6 @@
from setuptools import setup
setup(
name='release-action',
version='0.0.1.dev0',
)

0
src/__init__.py Normal file
View File

393
src/main.py Executable file
View File

@@ -0,0 +1,393 @@
import pickle
import re
from argparse import ArgumentParser
from dataclasses import replace
from typing import Optional
import yaml
from release import versioning
from release.context import ReleaseContext
from release.project import (ArtefactDescription, DeploymentCondition,
DeploymentDescription, HelmRelease, Npm, OciImage,
ProjectDescription, Sdist, Tarball, Wheel,
parse_project_description)
from release.release import (create_release, publish_artefacts,
update_deployments)
def load_project_description(filename) -> ProjectDescription:
with open(filename, 'rb') as f:
return pickle.load(f)
def save_project_description(filename, project_description):
with open(filename, 'wb') as f:
pickle.dump(project_description, f)
def make_project_description(
version_descriptor_filename: str) -> ProjectDescription:
return ProjectDescription(version_descriptor=version_descriptor_filename)
def make_context(gitea_instance: str,
repository_name: str,
ref_name: str,
run_number: str,
commit_sha: str,
is_pre_release: bool) -> ReleaseContext:
return ReleaseContext(gitea_instance=gitea_instance,
repository_name=repository_name,
ref_name=ref_name,
run_number=run_number,
commit_sha=commit_sha,
is_pre_release=is_pre_release)
def dump_project_description(project_description: ProjectDescription):
print('version_descriptor: %s' % project_description.version_descriptor)
print('project version: %s' % project_description.project_version)
print('planned version: %s' % project_description.planned_version)
print('is already released: %s' % project_description.is_released)
print('')
print('artefacts:')
for artefact in project_description.artefacts:
generated = artefact.generated
version_descriptor = (
artefact.version_descriptor
or project_description.version_descriptor)
release_info = generated.make_release_info(
project_description.context,
project_description.planned_version)
if isinstance(generated, OciImage):
print(' - oci image: %s' % generated.name)
print(' repository: %s' % generated.repository)
print(' local tag: %s' % release_info.local_tag)
print(' local name: %s' % release_info.local_full_name)
print(' remote names:')
for name in release_info.remote_full_names:
print(' - %s' % name)
print(' docker tags: %s' % ", ".join(release_info.tags))
elif isinstance(generated, Tarball):
print(' - tarball: %s' % generated.filename)
print(' package name: %s' % generated.package_name)
print(' repository: %s' % generated.repository)
print(' release version name: %s' % release_info.version_str)
elif isinstance(generated, Wheel):
print(' - wheel: %s' % generated.pattern)
print(' repository: %s' % generated.repository)
print(' release version name: %s' % release_info.version_str)
elif isinstance(generated, Sdist):
print(' - sdist: %s' % generated.filename)
print(' repository: %s' % generated.repository)
print(' release version name: %s' % release_info.version_str)
elif isinstance(generated, Npm):
print(' - npm: %s' % generated.directory)
print(' version_descriptor: %s' % version_descriptor)
print(' artefact version: %s' %
versioning.use_any(version_descriptor).version)
print('')
print('deployments:')
for desc in project_description.deployments:
deployment = desc.deployment
condition = desc.condition
if condition == DeploymentCondition.ALWAYS:
condition_str = 'always'
elif condition == DeploymentCondition.NEVER:
condition_str = 'never'
elif condition == DeploymentCondition.PRE_RELEASE_ONLY:
condition_str = 'pre_release_only'
elif condition == DeploymentCondition.RELEASE_ONLY:
condition_str = 'release_only'
print(' - condition: %s' % condition_str)
if isinstance(deployment, HelmRelease):
print(' type: helm release')
print(' release name: %s' % deployment.release_name)
print(' image paths: %s' % deployment.image_paths)
print(' namespace: %s' % deployment.namespace)
print(' repository: %s' % deployment.repository)
print('')
print('context:')
context = project_description.context
print(' gitea instance: %s' % context.gitea_instance)
print(' repository name: %s' % context.repository_name)
print(' ref name: %s' % context.ref_name)
print(' run number: %s' % context.run_number)
print(' commit sha: %s' % context.commit_sha)
print(' is pre-release: %s' % context.is_pre_release)
print('')
print('release info:')
release_info = project_description.release_info
print(' title: %s' % release_info.gitea_release_title)
print(' description: %s' % release_info.gitea_release_description)
print(' is prerelease: %s' % release_info.gitea_is_prerelease)
print(' git target commitish: %s' % release_info.gitea_git_commitish)
print(' git tags: %s' % ', '.join(release_info.git_tags))
print('')
print('environment variables:')
for key, value in project_description.environment_variables.items():
print(' %s: %s' % (key, value))
def make_artefact(type: str,
repository: str,
name: str,
filename: str,
package_name: str,
pattern: str,
directory: str,
version_descriptor) -> ArtefactDescription:
maybe_repository = ({'repository': repository}
if repository is not None
else {})
if type == 'oci_image':
assert name is not None
generated = OciImage(name=name, **maybe_repository)
elif type == 'tarball':
assert filename is not None
assert package_name is not None
generated = Tarball(filename=filename,
package_name=package_name,
**maybe_repository)
elif type == 'wheel':
assert pattern is not None
generated = Wheel(pattern=pattern, **maybe_repository)
elif type == 'sdist':
assert filename is not None
generated = Sdist(filename=filename, **maybe_repository)
elif type == 'npm':
assert directory is not None
generated = Npm(directory=directory)
else:
raise Exception('unknown artefact type: %s' % type)
return ArtefactDescription(
generated=generated, version_descriptor=version_descriptor)
def make_deployment(type: str,
release_name: str,
image_paths: list[str],
namespace: Optional[str],
repository: Optional[str],
condition_str: Optional[str]) -> DeploymentDescription:
if type == 'helm_release':
maybe_image_paths = ({'image_paths': image_paths}
if len(image_paths) > 0
else {})
maybe_repository = ({'repository': repository}
if repository is not None
else {})
deployment = HelmRelease(release_name=release_name,
namespace=(namespace or release_name),
**maybe_image_paths,
**maybe_repository)
else:
raise Exception('unknown deployment type: %s' % type)
if condition_str == 'always':
condition = DeploymentCondition.ALWAYS
elif condition_str == 'never':
condition = DeploymentCondition.NEVER
elif condition_str == 'pre_release_only':
condition = DeploymentCondition.PRE_RELEASE_ONLY
elif condition_str == 'release_only':
condition = DeploymentCondition.RELEASE_ONLY
elif condition_str is None:
condition = None
else:
raise Exception('unknown condition: %s' % condition_str)
maybe_condition = ({'condition': condition}
if condition is not None
else {})
return DeploymentDescription(deployment=deployment, **maybe_condition)
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument('action', choices=[
# TODO missing: adjust version (development)
# TODO missing: check if release already exists
'declare',
'check',
'add-artefact',
'add-deployment',
'publish-artefacts',
'update-deployments',
'create-release',
'dump'
])
def nullable_string(val):
if not val:
return None
return val
def space_separated(val):
return list(filter(lambda it: it != '', (val or '').split(' ')))
def true_or_false(val):
if val == '0' or val == 'false':
return False
elif val == '1' or val == 'true':
return True
else:
raise Exception('flag can be "0" or "1". got: %s' % val)
parser.add_argument('--state', required=True)
parser.add_argument('--version-descriptor', type=nullable_string)
parser.add_argument('--release-yaml-filename', type=nullable_string)
parser.add_argument('--dry-run', type=true_or_false)
parser.add_argument('--gitea-instance')
parser.add_argument('--release-repository-name')
parser.add_argument('--release-ref-name')
parser.add_argument('--release-run-number')
parser.add_argument('--release-commit-sha')
parser.add_argument('--is-pre-release', type=true_or_false)
parser.add_argument(
'--artefact-type', type=nullable_string,
choices=['oci_image', 'tarball', 'wheel', 'sdist', 'npm'])
parser.add_argument('--artefact-repository', type=nullable_string)
parser.add_argument('--artefact-name', type=nullable_string)
parser.add_argument('--artefact-package-name', type=nullable_string)
parser.add_argument('--artefact-filename', type=nullable_string)
parser.add_argument('--artefact-pattern', type=nullable_string)
parser.add_argument('--artefact-directory', type=nullable_string)
parser.add_argument('--deployment-type',
type=nullable_string,
choices=['helm_release'])
parser.add_argument('--deployment-release-name', type=nullable_string)
parser.add_argument('--deployment-image-paths', type=space_separated)
parser.add_argument('--deployment-namespace', type=nullable_string)
parser.add_argument('--deployment-repository', type=nullable_string)
parser.add_argument('--deployment-condition', type=nullable_string)
parser.add_argument('--write-env-vars-to-filename')
args = parser.parse_args()
def clean_repository_name(name: str) -> str:
if name.startswith('//'):
return re.match('^[/][/][^/]+[/](.+)$', name).group(1)
else:
return name
if args.action == 'declare':
if args.release_yaml_filename is None:
project_description = make_project_description(
args.version_descriptor)
else:
with open(args.release_yaml_filename, 'r') as f:
project_description = parse_project_description(
yaml.safe_load(f))
project_description = replace(
project_description,
context=make_context(
args.gitea_instance,
clean_repository_name(args.release_repository_name),
args.release_ref_name,
args.release_run_number,
args.release_commit_sha,
args.is_pre_release))
save_project_description(args.state, project_description)
elif args.action == 'add-artefact':
project_description = load_project_description(args.state)
artefact = make_artefact(args.artefact_type,
args.artefact_repository,
args.artefact_name,
args.artefact_filename,
args.artefact_package_name,
args.artefact_pattern,
args.artefact_directory,
args.version_descriptor)
project_description = replace(
project_description,
artefacts=project_description.artefacts + [artefact])
save_project_description(args.state, project_description)
elif args.action == 'add-deployment':
project_description = load_project_description(args.state)
deployment = make_deployment(args.deployment_type,
args.deployment_release_name,
args.deployment_image_paths,
args.deployment_namespace,
args.deployment_repository,
args.deployment_condition)
project_description = replace(
project_description,
deployments=project_description.deployments + [deployment])
save_project_description(args.state, project_description)
elif args.action == 'publish-artefacts':
project_description = load_project_description(args.state)
publish_artefacts(project_description, args.dry_run)
elif args.action == 'update-deployments':
project_description = load_project_description(args.state)
update_deployments(project_description, args.dry_run)
elif args.action == 'create-release':
project_description = load_project_description(args.state)
create_release(project_description, args.dry_run)
elif args.action == 'dump':
project_description = load_project_description(args.state)
dump_project_description(project_description)
elif args.action == 'check':
project_description = load_project_description(args.state)
else:
raise NotImplementedError()
assert project_description is not None
env_var_filename = args.write_env_vars_to_filename
if env_var_filename is not None:
env_vars = project_description.environment_variables
assert not any(map(lambda v: '"' in v, env_vars.values()))
with open(env_var_filename, 'a') as f:
f.write(
'\n'.join(map(lambda it: '%s="%s"' % it, env_vars.items())))

0
src/release/__init__.py Normal file
View File

16
src/release/context.py Normal file
View File

@@ -0,0 +1,16 @@
from dataclasses import dataclass
@dataclass(frozen=True)
class ReleaseContext:
gitea_instance: str
repository_name: str
ref_name: str
run_number: int
commit_sha: str
is_pre_release: bool
def make_environment_variables(self, version):
return {
'RELEASE_IS_PRERELEASE': ('1' if self.is_pre_release else '0'),
}

406
src/release/project.py Normal file
View File

@@ -0,0 +1,406 @@
import operator
from dataclasses import dataclass, field
from enum import Enum
from functools import cached_property, reduce
from itertools import chain
from typing import Optional, Union
from release import toolkit, versioning
from release.context import ReleaseContext
from release.versioning import Version
DEFAULT_OCI_IMAGE_REPOSITORY = 'europe-docker.pkg.dev/puzzle-and-play/docker'
DEFAULT_HELM_CHART_REPOSITORY = 'europe-docker.pkg.dev/puzzle-and-play/helm'
DEFAULT_GITEA_PACKAGE_INSTANCE = 'https://gitea.puzzleyou.net'
DEFAULT_PYPI_REPOSITORY_NAME = 'gitea'
HELM_DEFAULT_TIMEOUT = '5m'
def _normalize_env_var_fragment(txt: str) -> str:
return txt.upper().replace('-', '_').replace('.', '_')
def _python_version_str(version: Version) -> str:
return ('%d.%d.%d' % (version.major, version.minor, version.patch)) \
+ ('' if version.prerelease is None else '.%s' % version.prerelease)
@dataclass(frozen=True)
class OciImageReleaseInfo:
image_name: str
local_tag: str
local_full_name: str
remote_full_names: str
tags: list[str]
@dataclass(frozen=True)
class OciImage:
name: str
repository: str = DEFAULT_OCI_IMAGE_REPOSITORY
def make_environment_variables(self, context, version):
tag = context.commit_sha
normalized_image_name = _normalize_env_var_fragment(self.name)
release_info = self.make_release_info(context, version)
return {
'RELEASE_IMAGE_TAG': tag,
'RELEASE_IMAGE_LOCAL_NAME_%s' % normalized_image_name:
release_info.local_full_name,
}
def make_release_info(self, context: ReleaseContext, version: Version):
version_tag = 'v%s' % version
tags = ([version_tag, '%s.latest' % context.ref_name] + (
['development']
if context.is_pre_release
else [
'v%d.%d' % (version.major, version.minor),
'v%d' % version.major,
'latest',
]))
remote_full_names = list(
map(lambda tag: '%s/%s:%s' % (self.repository, self.name, tag),
tags))
return OciImageReleaseInfo(
image_name=self.name,
local_tag=context.commit_sha,
local_full_name='%s:%s' % (self.name, context.commit_sha),
remote_full_names=remote_full_names,
tags=tags)
@dataclass(frozen=True)
class TarballReleaseInfo:
filename: str
package_name: str
version_str: str
repository: str
@dataclass(frozen=True)
class Tarball:
filename: str
package_name: str
repository: str = DEFAULT_GITEA_PACKAGE_INSTANCE
def make_environment_variables(self, context, version):
return {}
def make_release_info(self, context: ReleaseContext, version: Version):
return TarballReleaseInfo(
filename=self.filename,
package_name=self.package_name,
repository=self.repository,
version_str=str(version))
@dataclass(frozen=True)
class WheelReleaseInfo:
pattern: str
repository: str
version_str: str
@dataclass(frozen=True)
class Wheel:
pattern: str
repository: str = DEFAULT_PYPI_REPOSITORY_NAME
def make_environment_variables(self, context, version):
return {}
def make_release_info(self, context: ReleaseContext, version: Version):
return WheelReleaseInfo(
pattern=self.pattern,
repository=self.repository,
version_str=_python_version_str(version))
@dataclass(frozen=True)
class SdistReleaseInfo:
filename: str
repository: str
version_str: str
@dataclass(frozen=True)
class Sdist:
filename: str
repository: str = DEFAULT_PYPI_REPOSITORY_NAME
def make_environment_variables(self, context, version):
return {}
def make_release_info(self, context: ReleaseContext, version: Version):
return SdistReleaseInfo(
filename=self.filename,
repository=self.repository,
version_str=_python_version_str(version))
@dataclass(frozen=True)
class NpmReleaseInfo:
directory: str
@dataclass(frozen=True)
class Npm:
directory: str
def make_environment_variables(self, context, version):
return {}
def make_release_info(self, context: ReleaseContext, version: Version):
return NpmReleaseInfo(directory=self.directory)
@dataclass(frozen=True)
class ArtefactDescription:
generated: Union[OciImage, Tarball, Wheel, Sdist, Npm]
version_descriptor: Optional[str] = None # if different from main
@dataclass(frozen=True)
class HelmReleaseInfo:
release_name: str
namespace: str
repository: str
image_tag: str
image_paths: str
timeout: str
@dataclass(frozen=True)
class HelmRelease:
release_name: str
namespace: str = None
image_paths: list[str] = field(default_factory=lambda: ['image.tag'])
repository: str = DEFAULT_HELM_CHART_REPOSITORY
def make_environment_variables(self, context, version):
return {}
def make_release_info(self, context: ReleaseContext, version: Version):
return HelmReleaseInfo(
release_name=self.release_name,
namespace=self.namespace or self.release_name,
repository=self.repository,
image_tag='v%s' % version,
image_paths=self.image_paths,
timeout=HELM_DEFAULT_TIMEOUT,
)
class DeploymentCondition(Enum):
ALWAYS = 0
NEVER = 1
PRE_RELEASE_ONLY = 2
RELEASE_ONLY = 3
@dataclass(frozen=True)
class DeploymentDescription:
deployment: Union[HelmRelease]
condition: DeploymentCondition = DeploymentCondition.PRE_RELEASE_ONLY
@dataclass(frozen=True)
class ProjectReleaseInfo:
gitea_release_title: str
gitea_release_description: str
gitea_is_prerelease: bool
gitea_git_commitish: str
git_tags: list[str]
@dataclass(frozen=True)
class ProjectDescription:
version_descriptor: str # filename
artefacts: list[ArtefactDescription] = field(default_factory=lambda: [])
deployments: list[DeploymentDescription] = field(
default_factory=lambda: [])
context: Optional[ReleaseContext] = None
@cached_property
def release_info(self):
if self.context is None:
return None
version = self.planned_version
return ProjectReleaseInfo(
gitea_release_title='Version %s' % version,
gitea_release_description='',
gitea_is_prerelease=self.context.is_pre_release,
gitea_git_commitish=self.context.commit_sha,
git_tags=['v%s' % version] + (
['development']
if self.context.is_pre_release
else [
'v%d.%d' % (version.major, version.minor),
'v%d' % version.major,
'latest',
]))
@cached_property
def planned_version(self):
if self.context is not None and self.context.is_pre_release:
prerelease_version = self.context.run_number or 0
return self.project_version.replace(
prerelease='dev%s' % prerelease_version)
else:
return self.project_version
@cached_property
def project_version(self):
return versioning.use_any(self.version_descriptor).version
@cached_property
def is_released(self):
if self.gitea_tool is None:
return None
return self.gitea_tool.is_released(self.planned_version)
@cached_property
def gitea_tool(self):
if self.context is None:
return None
if self.context.repository_name is None:
return None
if self.context.gitea_instance is None:
return None
return toolkit.Gitea(self.context.gitea_instance,
self.context.repository_name,
self.context.is_pre_release or False)
@property
def environment_variables(self):
version = self.planned_version
return reduce(
operator.or_,
chain(
map(lambda a:
a.generated.make_environment_variables(
self.context, version),
self.artefacts),
map(lambda d:
d.deployment.make_environment_variables(
self.context, version),
self.deployments),
([self.context.make_environment_variables(version)]
if self.context is not None
else []),
[{
'RELEASE_PROJECT_CURRENT_VERSION':
str(self.project_version),
'RELEASE_PROJECT_PLANNED_VERSION':
str(self.planned_version),
}]
))
def parse_project_description(obj):
def optional(obj, field, acc=lambda o, f: o[f]):
if field in obj:
return {field: acc(obj, field)}
else:
return {}
def parse_oci_image(img):
assert 'name' in img
return OciImage(name=img['name'],
**optional(img, 'repository'))
def parse_tarball(tar):
assert 'filename' in tar
return Tarball(filename=tar['filename'],
package_name=tar['package_name'],
**optional(tar, 'repository'))
def parse_wheel(whl):
assert 'pattern' in whl
return Wheel(pattern=whl['pattern'],
**optional(whl, 'repository'))
def parse_sdist(sdist):
assert 'filename' in sdist
return Sdist(filename=sdist['filename'],
**optional(sdist, 'repository'))
def parse_npm(npm):
assert 'directory' in npm
return Npm(directory=npm['directory'])
def parse_artefact(art):
assert 'type' in art
if art['type'] == 'oci_image' or art['type'] == 'docker':
generated = parse_oci_image(art)
elif art['type'] == 'tarball':
generated = parse_tarball(art)
elif art['type'] == 'wheel':
generated = parse_wheel(art)
elif art['type'] == 'sdist':
generated = parse_sdist(art)
elif art['type'] == 'npm':
generated = parse_npm(art)
else:
raise Exception('unknown artefact type: %s' % art['type'])
return ArtefactDescription(
generated=generated,
**optional(art, 'version_descriptor',))
def parse_helm(helm):
assert 'release_name' in helm
return HelmRelease(
release_name=helm['release_name'],
**optional(helm, 'namespace'),
**optional(helm, 'image_paths'),
**optional(helm, 'repository'))
def parse_condition(cnd, _):
assert 'condition' in cnd
txt = cnd['condition'].lower()
if txt == 'always':
return DeploymentCondition.ALWAYS
if txt == 'never':
return DeploymentCondition.NEVER
if txt == 'pre_release_only':
return DeploymentCondition.PRE_RELEASE_ONLY
if txt == 'release_only':
return DeploymentCondition.RELEASE_ONLY
else:
raise Exception('unknown condition: %s' % txt)
def parse_deployment(dpl):
assert 'type' in dpl
if dpl['type'] == 'helm':
deployment = parse_helm(dpl)
else:
raise Exception('unknown deployment type: %s' % dpl['type'])
return DeploymentDescription(
deployment=deployment,
**optional(dpl, 'condition', parse_condition))
assert 'version_descriptor' in obj
return ProjectDescription(
version_descriptor=obj['version_descriptor'],
artefacts=list(map(parse_artefact, obj.get('artefacts', []))),
deployments=list(map(parse_deployment, obj.get('deployments', []))),
)

233
src/release/release.py Normal file
View File

@@ -0,0 +1,233 @@
import json
import subprocess
from os import path
from tempfile import NamedTemporaryFile, TemporaryDirectory
from release.project import (DeploymentCondition, HelmRelease, HelmReleaseInfo,
Npm, NpmReleaseInfo, OciImage,
OciImageReleaseInfo, ProjectDescription, Sdist,
SdistReleaseInfo, Tarball, TarballReleaseInfo,
Wheel, WheelReleaseInfo)
class Cli:
def __init__(self, dry_run: bool):
self.dry_run = dry_run
def __call__(self, *cmd, cwd=None):
try:
cleaned_cmd = list(filter(lambda it: it != '', cmd))
if cwd is not None:
print('$> cd %s' % cwd)
print('$> %s' % ' '.join(cleaned_cmd))
if not self.dry_run:
res = subprocess.run(cleaned_cmd,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
output = res.stdout
else:
output = b''
for line in output.decode('utf-8').splitlines():
print(' | %s' % line)
return output
except subprocess.CalledProcessError as exc:
for line in exc.stdout.decode('utf-8').splitlines():
print('[ERR] %s' % line)
raise Exception('%s failed: %s' % (cleaned_cmd[0],
exc.stderr.decode('utf-8')))
def publish_oci_image(info: OciImageReleaseInfo, cli: Cli):
print('-- publishing oci image --')
print('name: %s' % info.image_name)
for remote_full_name in info.remote_full_names:
cli('docker', 'tag', info.local_full_name, remote_full_name)
cli('docker', 'push', remote_full_name)
print()
def publish_tarball(info: TarballReleaseInfo, cli: Cli):
print('-- publishing tarball --')
print('filename: %s' % info.filename)
cli('curl',
'--netrc',
'--fail-with-body',
'--upload-file', info.filename,
'-i',
'-X', 'PUT',
'%s/api/packages/puzzleYOU/generic/%s/%s/%s' % ( # TODO owner
info.repository,
info.package_name,
info.version_str,
path.basename(info.filename)))
print()
def publish_wheel(info: WheelReleaseInfo, cli: Cli):
print('-- publishing wheel --')
cli('twine',
'upload',
'--verbose',
'--repository', info.repository,
info.pattern)
print()
def publish_sdist(info: SdistReleaseInfo, cli: Cli):
print('-- publishing python source distribution --')
cli('twine',
'upload',
'--verbose',
'--repository', info.repository,
info.filename)
print()
def publish_npm(info: NpmReleaseInfo, cli: Cli):
print('-- publishing npm package --')
cli('npm', 'publish', cwd=info.directory)
print()
def publish_artefacts(project: ProjectDescription, dry_run: bool = False):
cli = Cli(dry_run)
context = project.context
planned_version = project.planned_version
for artefact in project.artefacts:
release_info = artefact.generated.make_release_info(
context, planned_version)
if isinstance(artefact.generated, OciImage):
publish_oci_image(release_info, cli)
elif isinstance(artefact.generated, Tarball):
publish_tarball(release_info, cli)
elif isinstance(artefact.generated, Wheel):
publish_wheel(release_info, cli)
elif isinstance(artefact.generated, Sdist):
publish_sdist(release_info, cli)
elif isinstance(artefact.generated, Npm):
publish_npm(release_info, cli)
else:
print('unknown artefact: %s' % artefact.generated)
raise Exception('cannot publish unknown artefact')
def update_helm_release(info: HelmReleaseInfo, cli: Cli):
ro_cli = Cli(dry_run=False)
print('-- updating helm release --')
metadata = json.loads(ro_cli('helm',
'get', 'metadata', info.release_name,
'-n', info.namespace, '-o', 'json').decode())
chart_name = metadata['chart']
chart_version = metadata['version']
with TemporaryDirectory() as td:
ro_cli('helm',
'pull', 'oci://%s/%s' % (info.repository, chart_name),
'--version', chart_version,
'-d', td)
chart_filename = '%s/%s-%s.tgz' % (td, chart_name, chart_version)
value_overrides = ','.join(
map(lambda t: '%s=%s' % (t, info.image_tag), info.image_paths))
cli('helm',
'upgrade', info.release_name,
chart_filename,
'--version', chart_version,
'--namespace', info.namespace,
'--reuse-values',
'--set', value_overrides,
'--timeout', info.timeout)
def update_deployments(project: ProjectDescription, dry_run: bool = False):
cli = Cli(dry_run)
context = project.context
planned_version = project.planned_version
for deployment in project.deployments:
release_info = deployment.deployment.make_release_info(
context, planned_version)
if deployment.condition == DeploymentCondition.NEVER:
continue
if (deployment.condition == DeploymentCondition.RELEASE_ONLY
and context.is_pre_release):
continue
if (deployment.condition == DeploymentCondition.PRE_RELEASE_ONLY
and not context.is_pre_release):
continue
if isinstance(deployment.deployment, HelmRelease):
update_helm_release(release_info, cli)
else:
print('unknown deployment: %s' % deployment.deployment)
raise Exception('cannot publish unknown deployment')
def create_release(project: ProjectDescription, dry_run: bool = False):
cli = Cli(dry_run)
info = project.release_info
context = project.context
assert len(info.git_tags) > 0
# https://gitea.com/api/swagger#/repository/repoCreateRelease
payload = {
'body': info.gitea_release_description,
'draft': False,
'name': info.gitea_release_title,
'prerelease': info.gitea_is_prerelease,
'tag_message': info.gitea_release_title,
'tag_name': info.git_tags[0],
'target_commitish': info.gitea_git_commitish,
}
print('-- creating gitea release --')
print('POST body:')
print(json.dumps(payload, indent=2))
with NamedTemporaryFile(mode='w') as tf:
json.dump(payload, tf)
tf.flush()
cli('curl',
'--netrc',
'--fail-with-body',
'-i',
'-H', 'Content-Type: application/json',
'--data', '@%s' % tf.name,
'-X', 'POST',
'%s/api/v1/repos/%s/releases' % (context.gitea_instance,
context.repository_name))
print()
print('-- adding git tags --')
for tag in info.git_tags[1:]:
cli('git', 'tag', '-f', tag, info.gitea_git_commitish)
cli('git', 'push', '-f', 'origin', '--tags')

View File

View File

@@ -0,0 +1,41 @@
[package]
name = "resi"
version = "0.0.1"
edition = "2021"
description = "webservice for storing and delivering images"
[dependencies]
async-std.workspace = true
async-trait.workspace = true
serde.workspace = true
derive_more.workspace = true
thiserror.workspace = true
resi-aux.workspace = true
resi-core.workspace = true
resi-filters.workspace = true
resi-utils.workspace = true
resi-import.workspace = true
pyru.workspace = true
anyhow.workspace = true
tokio.workspace = true
futures.workspace = true
hyper.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
tracing-subscriber.workspace = true
tracing.workspace = true
bytes.workspace = true
url.workspace = true
regex.workspace = true
reqwest.workspace = true
tikv-jemallocator.workspace = true
sqlx.workspace = true
log.workspace = true
chrono.workspace = true
yolo-rs.workspace = true
image.workspace = true
arcstr.workspace = true
ndarray.workspace = true
once_cell.workspace = true
ort = "=2.0.0-rc.9"
ort-sys = "=2.0.0-rc.9"

View File

@@ -0,0 +1,65 @@
{
"name": "product-designer",
"version": "42.13.37",
"description": "",
"license": "",
"scripts": {
"ng": "ng",
"start": "ng serve",
"build": "ng build",
"test": "ng test",
"lint": "ng lint",
"e2e": "ng e2e"
},
"private": true,
"dependencies": {
"@angular/animations": "^5.0.0",
"@angular/cdk": "^5.0.2",
"@angular/common": "^5.0.0",
"@angular/compiler": "^5.0.0",
"@angular/core": "^5.0.0",
"@angular/forms": "^5.0.0",
"@angular/http": "^5.0.0",
"@angular/material": "^5.0.2",
"@angular/platform-browser": "^5.0.0",
"@angular/platform-browser-dynamic": "^5.0.0",
"@angular/platform-server": "^5.0.0",
"@angular/router": "^5.0.0",
"@iframe-resizer/child": "^5.4.6",
"bowser": "^2.11.0",
"core-js": "^2.5.1",
"crypto-js": "^3.1.9-1",
"dragdroptouch": "github:mychiara/dragdroptouch#master",
"enhanced-resolve": "^3.1.0",
"fastest-levenshtein": "^1.0.16",
"gl-matrix": "^2.6.1",
"hammerjs": "^2.0.8",
"ngx-pagination": "^3.0.1",
"rxjs": "^5.5.2",
"svg.js": "^2.6.3",
"text-encoding": "^0.6.4",
"xmlserializer": "^0.6.0",
"zone.js": "^0.11.4"
},
"devDependencies": {
"@angular/cli": "~1.7.4",
"@angular/compiler-cli": "^5.2.11",
"@types/jasmine": "2.5.54",
"@types/node": "^7.0.43",
"codelyzer": "~3.0.1",
"jasmine-core": "~5.1.2",
"jasmine-spec-reporter": "~7.0.0",
"karma": "~6.4.3",
"karma-chrome-launcher": "^3.2.0",
"karma-cli": "~2.0.0",
"karma-coverage-istanbul-reporter": "^3.0.3",
"karma-firefox-launcher": "^1.1.0",
"karma-jasmine": "~5.1.0",
"karma-jasmine-html-reporter": "^2.1.0",
"node-sass": "^9.0.0",
"protractor": "~5.1.2",
"ts-node": "~3.0.6",
"tslint": "~5.2.0",
"typescript": "~2.6.1"
}
}

View File

@@ -0,0 +1,19 @@
[build-system]
requires = ["maturin>=1"]
build-backend = "maturin"
[project]
name = "prngl"
version = "0.0.3"
requires-python = ">=3.7"
classifiers = [
"Programming Language :: Rust",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
]
dependencies = [
'resi',
]
[tool.maturin]
python-source = "python"

View File

@@ -0,0 +1,50 @@
version_descriptor: 'test-assets/version.txt'
artefacts:
- type: oci_image
name: productdesignerd
- type: docker
name: motacilla
version_descriptor: 'test-assets/Cargo.toml'
repository: remote
- type: tarball
package_name: 'yaac'
filename: '/tmp/yaac.tar.gz'
- type: tarball
package_name: 'yaac'
filename: '/tmp/yaac.tar.gz'
repository: balls
- type: wheel
pattern: './scratch/wheels/*.whl'
- type: wheel
pattern: './scratch/wheels/*.whl'
repository: other
- type: sdist
filename: './dist/papyru-0.0.1.tar.gz'
- type: sdist
filename: './dist/papyru-0.0.1.tar.gz'
repository: 'pypi'
- type: npm
directory: ./dist/browser
deployments:
- type: helm
release_name: 'productdesignerd-testing'
- type: helm
condition: release_only
release_name: foo
namespace: bar
image_paths:
- foo
- bar
- baz
repository: helms

View File

@@ -0,0 +1,43 @@
from setuptools import setup
setup(
name='papyru',
version='2.10.4',
description=(
'minimal REST library with OpenAPI-based validation for django'),
author='puzzleYOU GmbH',
author_email='papyru@puzzleyou.net',
url='http://www.puzzleyou.net/',
license='AGPLv3',
platforms=['any'],
packages=[
'papyru',
'papyru.static',
'papyru.varspool',
'papyru.varspool.command'
],
package_data={
'papyru.varspool': ['assets/*'],
},
install_requires=[
'Cerberus',
'Django',
'jsonschema',
'pyyaml',
'requests',
'lxml',
'python-dateutil',
],
classifiers=[
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'Topic :: Internet :: WWW/HTTP',
'License :: OSI Approved :: GNU Affero General Public License v3',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
],
scripts=[
'bin/generate_jsonschema.py'
]
)

View File

@@ -0,0 +1 @@
3.14.15

View File

@@ -0,0 +1,13 @@
from unittest import TestCase
from release.context import ReleaseContext
class TestReleaseContext(TestCase):
def test_can_describe_context(self):
ReleaseContext(repository_name='resi',
ref_name='testing',
run_number=42,
is_pre_release=True,
commit_sha='0AB123',
gitea_instance='http://gitea.foo.intern')

View File

@@ -0,0 +1,389 @@
from os import path
from unittest import TestCase
import yaml
from semver import Version
from release.context import ReleaseContext
from release.project import (ArtefactDescription, DeploymentCondition,
DeploymentDescription, HelmRelease,
HelmReleaseInfo, Npm, NpmReleaseInfo, OciImage,
OciImageReleaseInfo, ProjectDescription,
ProjectReleaseInfo, Sdist, SdistReleaseInfo,
Tarball, TarballReleaseInfo, Wheel,
WheelReleaseInfo, parse_project_description)
class TestProjectDescription(TestCase):
def test_can_describe_projects(self):
# resi-lib
ProjectDescription(
version_descriptor='src/python/Cargo.toml',
artefacts=[
ArtefactDescription(
generated=Wheel(pattern='./scratch/wheels/*.whl'))
])
# productdesignerd
ProjectDescription(
version_descriptor='bootstrap/setup.py',
artefacts=[
ArtefactDescription(
generated=OciImage(name='productdesignerd')
)],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='productdesignerd-testing',
namespace='productdesignerd-testing')),
])
# masa-images
ProjectDescription(
version_descriptor='Cargo.toml',
artefacts=[
ArtefactDescription(
generated=OciImage(name='masa-images')),
ArtefactDescription(
generated=Wheel(pattern='./scratch/wheels/*.whl'))
],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='masa-images-testing',
namespace='masa-images-testing')),
])
# yaac
ProjectDescription(
version_descriptor='version.txt',
artefacts=[
ArtefactDescription(generated=Tarball(
filename='/tmp/yaac.tar.gz',
package_name='yaac'))
])
# papyru
ProjectDescription(
version_descriptor='setup.py',
artefacts=[
ArtefactDescription(
generated=Sdist(
'./dist/papyru-${PAPYRU_PACKAGE_VERSION}.tar.gz'))
])
# productdesigner
ProjectDescription(
version_descriptor='package.json',
artefacts=[
ArtefactDescription(
generated=Npm('dist/browser')),
ArtefactDescription(
generated=OciImage(name='productdesigner')),
],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='productdesigner-testing',
namespace='productdesigner-testing'))
])
# motacilla
ProjectDescription(
version_descriptor='setup.py',
artefacts=[
ArtefactDescription(generated=OciImage(name='motacilla')),
ArtefactDescription(generated=OciImage(name='motacilla-cdn')),
],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='motacilla-de-testing',
namespace='motacilla-de-testing',
image_paths=['image.cms.tag', 'image.cdn.tag'])),
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='motacilla-schmidt-testing',
namespace='motacilla-schmidt-testing',
image_paths=['image.cms.tag', 'image.cdn.tag'])),
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='motacilla-be-testing',
namespace='motacilla-be-testing',
image_paths=['image.cms.tag', 'image.cdn.tag'])),
])
# prngl
ProjectDescription(
version_descriptor='Cargo.toml',
artefacts=[
ArtefactDescription(
version_descriptor='src/python/pyproject.toml',
generated=Wheel(pattern='./scratch/wheels/*.whl')),
ArtefactDescription(generated=OciImage(name='prngl')),
],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='prngl-testing',
namespace='prngl-testing'))
])
def test_environment_variables(self):
# motacilla
desc = ProjectDescription(
version_descriptor=path.join(
path.dirname(__file__), 'assets/setup.py'),
artefacts=[
ArtefactDescription(generated=OciImage(
repository='remote', name='motacilla')),
ArtefactDescription(generated=OciImage(
repository='remote', name='motacilla-cdn')),
],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='motacilla-de-testing',
namespace='motacilla-de-testing',
image_paths=['image.cms.tag', 'image.cdn.tag'])),
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='motacilla-schmidt-testing',
namespace='motacilla-schmidt-testing',
image_paths=['image.cms.tag', 'image.cdn.tag'])),
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='motacilla-be-testing',
namespace='motacilla-be-testing',
image_paths=['image.cms.tag', 'image.cdn.tag'])),
],
context=ReleaseContext(repository_name='resi',
ref_name='testing',
run_number=42,
is_pre_release=True,
commit_sha='0AB123',
gitea_instance='http://gitea.foo.intern')
)
self.maxDiff = None
self.assertEqual(
desc.environment_variables,
{
'RELEASE_IMAGE_TAG': '0AB123',
'RELEASE_IMAGE_LOCAL_NAME_MOTACILLA': 'motacilla:0AB123',
'RELEASE_IMAGE_LOCAL_NAME_MOTACILLA_CDN':
'motacilla-cdn:0AB123',
'RELEASE_IS_PRERELEASE': '1',
'RELEASE_PROJECT_CURRENT_VERSION': '2.10.4',
'RELEASE_PROJECT_PLANNED_VERSION': '2.10.4-dev42'
}
)
def test_tarball_release_info(self):
tarball = Tarball(filename='foo.tar.gz', package_name='foo')
self.assertEqual(
TarballReleaseInfo(
filename='foo.tar.gz',
package_name='foo',
version_str='1.2.3-dev4',
repository='https://gitea.puzzleyou.net'),
tarball.make_release_info(None, Version(1, 2, 3, 'dev4')))
def test_wheel_release_info(self):
wheel = Wheel(pattern='dist/wheels/*')
self.assertEqual(
WheelReleaseInfo(pattern='dist/wheels/*',
repository='gitea',
version_str='1.2.3.dev4'),
wheel.make_release_info(None, Version(1, 2, 3, 'dev4')))
def test_sdist_release_info(self):
sdist = Sdist(filename='somethingsomething.tar.gz')
self.assertEqual(
SdistReleaseInfo(version_str='1.2.3.dev4',
filename='somethingsomething.tar.gz',
repository='gitea'),
sdist.make_release_info(None, Version(1, 2, 3, 'dev4')))
def test_npm_release_info(self):
npm = Npm(directory='./')
self.assertEqual(
NpmReleaseInfo(directory='./'),
npm.make_release_info(None, Version(1, 2, 3, 'dev4')))
def test_helm_release_info(self):
helm = HelmRelease(release_name='foo',
namespace='bar',
image_paths=['foo.bar', 'baz.blubb'],
repository='ludicrous')
self.assertEqual(
HelmReleaseInfo(release_name='foo',
namespace='bar',
repository='ludicrous',
image_tag='v1.2.3-dev4',
image_paths=['foo.bar', 'baz.blubb'],
timeout='5m'),
helm.make_release_info(None, Version(1, 2, 3, 'dev4')))
def test_oci_image_release_info(self):
self.maxDiff = None
img = OciImage(name='test', repository='remote')
pre_release_version = Version(1, 33, 7, 'dev42')
pre_release_context = ReleaseContext(
repository_name='resi',
ref_name='testing',
run_number=42,
is_pre_release=True,
commit_sha='PROBABLY_BROKEN',
gitea_instance='http://gitea.foo.intern')
self.assertEqual(
OciImageReleaseInfo(
image_name='test',
local_tag='PROBABLY_BROKEN',
local_full_name='test:PROBABLY_BROKEN',
remote_full_names=[
'remote/test:v1.33.7-dev42',
'remote/test:testing.latest',
'remote/test:development',
],
tags=['v1.33.7-dev42', 'testing.latest', 'development']),
img.make_release_info(pre_release_context,
pre_release_version))
release_version = Version(1, 33, 7)
release_context = ReleaseContext(
repository_name='resi',
ref_name='master',
run_number=43,
is_pre_release=False,
commit_sha='finalv2',
gitea_instance='http://gitea.foo.intern')
self.assertEqual(
OciImageReleaseInfo(
image_name='test',
local_tag='finalv2',
local_full_name='test:finalv2',
remote_full_names=[
'remote/test:v1.33.7',
'remote/test:master.latest',
'remote/test:v1.33',
'remote/test:v1',
'remote/test:latest',
],
tags=['v1.33.7',
'master.latest',
'v1.33',
'v1',
'latest']),
img.make_release_info(release_context, release_version))
def test_project_release_info(self):
project_0 = ProjectDescription(version_descriptor='version.txt')
self.assertIsNone(project_0.release_info)
project_pre = ProjectDescription(
version_descriptor='version.txt',
context=ReleaseContext(
repository_name='resi',
ref_name='testing',
run_number=42,
is_pre_release=True,
commit_sha='PROBABLY_BROKEN',
gitea_instance='http://gitea.foo.intern'))
self.assertEqual(
ProjectReleaseInfo(
gitea_release_title='Version 0.0.1-dev42',
gitea_release_description='',
gitea_is_prerelease=True,
gitea_git_commitish='PROBABLY_BROKEN',
git_tags=['v0.0.1-dev42', 'development'],
),
project_pre.release_info)
project = ProjectDescription(
version_descriptor='version.txt',
context=ReleaseContext(
repository_name='resi',
ref_name='master',
run_number=42,
is_pre_release=False,
commit_sha='PROBABLY_BROKEN',
gitea_instance='http://gitea.foo.intern'))
self.assertEqual(
ProjectReleaseInfo(
gitea_release_title='Version 0.0.1',
gitea_release_description='',
gitea_is_prerelease=False,
gitea_git_commitish='PROBABLY_BROKEN',
git_tags=['v0.0.1', 'v0.0', 'v0', 'latest'],
),
project.release_info)
def test_load_from_file(self):
expected = ProjectDescription(
version_descriptor='test-assets/version.txt',
artefacts=[
ArtefactDescription(generated=OciImage(
name='productdesignerd')),
ArtefactDescription(
generated=OciImage(repository='remote', name='motacilla'),
version_descriptor='test-assets/Cargo.toml'),
ArtefactDescription(generated=Tarball(
filename='/tmp/yaac.tar.gz', package_name='yaac')),
ArtefactDescription(generated=Tarball(
filename='/tmp/yaac.tar.gz',
package_name='yaac',
repository='balls')),
ArtefactDescription(generated=Wheel(
pattern='./scratch/wheels/*.whl')),
ArtefactDescription(generated=Wheel(
pattern='./scratch/wheels/*.whl', repository='other')),
ArtefactDescription(generated=Sdist(
filename='./dist/papyru-0.0.1.tar.gz')),
ArtefactDescription(generated=Sdist(
filename='./dist/papyru-0.0.1.tar.gz',
repository='pypi')),
ArtefactDescription(generated=Npm(directory='./dist/browser')),
],
deployments=[
DeploymentDescription(
condition=DeploymentCondition.PRE_RELEASE_ONLY,
deployment=HelmRelease(
release_name='productdesignerd-testing')),
DeploymentDescription(
condition=DeploymentCondition.RELEASE_ONLY,
deployment=HelmRelease(
release_name='foo',
namespace='bar',
image_paths=['foo', 'bar', 'baz'],
repository='helms',
)),
])
with open(path.join(
path.dirname(__file__), 'assets/release.yaml'), 'r') as f:
project = parse_project_description(yaml.safe_load(f))
self.maxDiff = None
self.assertEqual(project, expected)

View File

@@ -0,0 +1,66 @@
from os import path
from tempfile import NamedTemporaryFile
from unittest import TestCase
from semver import Version
from release.versioning import use_any
def _asset_path(filename):
return path.join(path.dirname(__file__), 'assets/', filename)
def _test_can_read_version(test, filename, expected):
v = use_any(_asset_path(filename))
test.assertEqual(v.version, expected)
def _test_can_write_version(test, filename):
v = use_any(_asset_path(filename))
v.version = Version(1, 33, 7)
with NamedTemporaryFile() as tf:
v.store(tf.name)
v2 = use_any(tf.name)
test.assertEqual(v2.version, Version(1, 33, 7))
class TestSetupPy(TestCase):
def test_can_read_version(self):
_test_can_read_version(self, 'setup.py', Version(2, 10, 4))
def test_can_write_version(self):
_test_can_write_version(self, 'setup.py')
class TestCargoToml(TestCase):
def test_can_read_version(self):
_test_can_read_version(self, 'Cargo.toml', Version(0, 0, 1))
def test_can_write_version(self):
_test_can_write_version(self, 'Cargo.toml')
class TestPyProjectToml(TestCase):
def test_can_read_version(self):
_test_can_read_version(self, 'pyproject.toml', Version(0, 0, 3))
def test_can_write_version(self):
_test_can_write_version(self, 'pyproject.toml')
class TestPackageJson(TestCase):
def test_can_read_version(self):
_test_can_read_version(self, 'package.json', Version(42, 13, 37))
def test_can_write_version(self):
_test_can_write_version(self, 'package.json')
class TestVersionTxt(TestCase):
def test_can_read_version(self):
_test_can_read_version(self, 'version.txt', Version(3, 14, 15))
def test_can_write_version(self):
_test_can_write_version(self, 'version.txt')

36
src/release/toolkit.py Normal file
View File

@@ -0,0 +1,36 @@
import requests
class Gitea:
def __init__(self,
gitea_instance: str,
repository_name: str,
is_pre_release: bool):
self.instance = gitea_instance
self.repository_name = repository_name
self.is_pre_release = is_pre_release
def is_released(self, version):
resp = requests.get(
'%s/repos/%s/releases/tags/%s' % (self.api_url,
self.repository_name,
version_to_git_tag(version)))
if resp.status_code == 404:
return False
resp.raise_for_status()
return True
def api_get(self, endpoint):
resp = requests.get('%s/%s' % (self.api_url, endpoint))
resp.raise_for_status()
return resp.json()
@property
def api_url(self):
return '%s/api/v1' % self.instance
def version_to_git_tag(version):
return 'v%s' % version

110
src/release/versioning.py Normal file
View File

@@ -0,0 +1,110 @@
import json
import re
from copy import deepcopy
from logging import getLogger
import toml
from semver import Version
logger = getLogger(__name__)
RE_SETUP_PY = re.compile(r'version\s?=\s?[\'"](.*)[\'"]')
class SetupPy:
def __init__(self, filename):
logger.warning('setup.py is discouraged. Use pyproject.toml.')
with open(filename, 'r') as f:
content = f.read()
version_string = RE_SETUP_PY.search(content)
if version_string is None:
raise Exception('could not find version in %s' % filename)
self.filename = filename
self.content = content
self.version = Version.parse(version_string.group(1))
def store(self, destination=None):
destination = destination or self.filename
edited = RE_SETUP_PY.sub(
'version=\'%s\'' % self.version, self.content)
with open(destination, 'w') as f:
f.write(edited)
class Structured:
def __init__(self, filename, format, item_path):
with open(filename, 'r') as f:
obj = format.load(f)
cur = obj
for part in item_path:
cur = cur[part]
self.format = format
self.item_path = item_path
self.content = obj
self.version = Version.parse(cur)
def store(self, destination=None):
destination = destination or self.filename
edited = deepcopy(self.content)
cur = edited
for part in self.item_path[:-1]:
cur = cur[part]
cur[self.item_path[-1]] = str(self.version)
with open(destination, 'w') as f:
self.format.dump(edited, f)
class VersionTxt:
def __init__(self, filename):
self.filename = filename
with open(filename, 'r') as f:
self.version = Version.parse(f.read())
def store(self, destination=None):
destination = destination or self.filename
with open(destination, 'w') as f:
f.write(str(self.version))
def use_pyproject_toml(filename):
return Structured(filename, toml, ['project', 'version'])
def use_cargo_toml(filename):
return Structured(filename, toml, ['package', 'version'])
def use_package_json(filename):
return Structured(filename, json, ['version'])
def use_setup_py(filename):
return SetupPy(filename)
def use_version_txt(filename):
return VersionTxt(filename)
def use_any(filename):
for f in [use_pyproject_toml,
use_cargo_toml,
use_package_json,
use_version_txt,
use_setup_py]:
try:
return f(filename)
except Exception:
pass
raise Exception('cannot detect format of %s' % filename)

10
src/test.py Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python3
import logging
import unittest
logging.basicConfig(level=logging.ERROR)
from release.tests.context import * # noqa
from release.tests.project import * # noqa
from release.tests.versioning import * # noqa
unittest.main()

41
test-assets/Cargo.toml Normal file
View File

@@ -0,0 +1,41 @@
[package]
name = "resi"
version = "0.0.1"
edition = "2021"
description = "webservice for storing and delivering images"
[dependencies]
async-std.workspace = true
async-trait.workspace = true
serde.workspace = true
derive_more.workspace = true
thiserror.workspace = true
resi-aux.workspace = true
resi-core.workspace = true
resi-filters.workspace = true
resi-utils.workspace = true
resi-import.workspace = true
pyru.workspace = true
anyhow.workspace = true
tokio.workspace = true
futures.workspace = true
hyper.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
tracing-subscriber.workspace = true
tracing.workspace = true
bytes.workspace = true
url.workspace = true
regex.workspace = true
reqwest.workspace = true
tikv-jemallocator.workspace = true
sqlx.workspace = true
log.workspace = true
chrono.workspace = true
yolo-rs.workspace = true
image.workspace = true
arcstr.workspace = true
ndarray.workspace = true
once_cell.workspace = true
ort = "=2.0.0-rc.9"
ort-sys = "=2.0.0-rc.9"

1
test-assets/version.txt Normal file
View File

@@ -0,0 +1 @@
1.33.7

1
version.txt Normal file
View File

@@ -0,0 +1 @@
0.0.1