Загрузка данных


#!/usr/bin/env python3

"""
Universal build script

Pipeline example:

python build.py \
    --env prod \
    --git-ref v1.2.3

Requirements:

pip install hvac jinja2
"""

import argparse
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import tomllib

import hvac
from jinja2 import Environment, FileSystemLoader

###############################################################################
# CONFIG
###############################################################################

PROJECT_NAME = "mailarchive"

REPO_URL = "https://dev.azure.com/company/project/_git/mailarchive"

REGISTRY = "registry.company.ru"

VAULT_PATHS = {
    "dev": "projects/mailarchive/dev",
    "stage": "projects/mailarchive/stage",
    "prod": "projects/mailarchive/prod",
}

###############################################################################
# LOGGING
###############################################################################


logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s %(levelname)s %(message)s"
)

logger = logging.getLogger(__name__)

###############################################################################
# HELPERS
###############################################################################


def run(cmd, cwd=None):
    """
    Execute shell command.
    Exit immediately on failure.
    """

    logger.info("RUN: %s", " ".join(cmd))

    subprocess.run(
        cmd,
        cwd=cwd,
        check=True
    )


###############################################################################
# GIT
###############################################################################


def clone_repository(workdir):
    """
    Clone repository to temporary directory.
    """

    repo_dir = os.path.join(workdir, PROJECT_NAME)

    run([
        "git",
        "clone",
        REPO_URL,
        repo_dir
    ])

    return repo_dir


def checkout_git_ref(repo_dir, git_ref):
    """
    Checkout tag/branch/commit.
    """

    run([
        "git",
        "checkout",
        git_ref
    ], cwd=repo_dir)


###############################################################################
# VERSION
###############################################################################


def get_version(repo_dir):
    """
    Read version from pyproject.toml
    """

    pyproject = os.path.join(
        repo_dir,
        "pyproject.toml"
    )

    with open(pyproject, "rb") as f:
        data = tomllib.load(f)

    version = data["project"]["version"]

    logger.info("Version: %s", version)

    return version


###############################################################################
# VAULT
###############################################################################


def load_secrets(environment):
    """
    Load secrets from Vault.

    Required env vars:

        VAULT_ADDR
        VAULT_TOKEN
    """

    logger.info(
        "Loading secrets from Vault (%s)",
        environment
    )

    client = hvac.Client(
        url=os.environ["VAULT_ADDR"],
        token=os.environ["VAULT_TOKEN"]
    )

    if not client.is_authenticated():
        raise RuntimeError(
            "Vault authentication failed"
        )

    path = VAULT_PATHS[environment]

    response = (
        client.secrets.kv.v2.read_secret_version(
            path=path
        )
    )

    secrets = response["data"]["data"]

    logger.info(
        "Loaded %s secret keys",
        len(secrets)
    )

    return secrets


###############################################################################
# CONFIG GENERATION
###############################################################################


def render_templates(repo_dir, secrets):
    """
    Render config templates.

    Example:

        templates/vhost.conf.j2
            ->
        build/config/vhost.conf
    """

    template_dir = os.path.join(
        repo_dir,
        "templates"
    )

    output_dir = os.path.join(
        repo_dir,
        "build",
        "config"
    )

    os.makedirs(output_dir, exist_ok=True)

    env = Environment(
        loader=FileSystemLoader(template_dir)
    )

    #
    # Example Apache vhost template
    #
    template = env.get_template(
        "vhost.conf.j2"
    )

    rendered = template.render(
        server_name=secrets.get("server_name"),
        ldap_password=secrets.get("ldap_password"),
        db_password=secrets.get("db_password"),
    )

    with open(
        os.path.join(output_dir, "vhost.conf"),
        "w"
    ) as f:
        f.write(rendered)

    logger.info("Templates rendered")


###############################################################################
# FILE SECRETS
###############################################################################


def write_secret_files(repo_dir, secrets):
    """
    If Vault stores certificates, keytabs,
    htpasswd etc.
    """

    secrets_dir = os.path.join(
        repo_dir,
        "build",
        "secrets"
    )

    os.makedirs(secrets_dir, exist_ok=True)

    #
    # Example:
    # keytab stored in Vault as base64
    #

    # import base64
    #
    # with open(
    #     f"{secrets_dir}/service.keytab",
    #     "wb"
    # ) as f:
    #     f.write(
    #         base64.b64decode(
    #             secrets["keytab"]
    #         )
    #     )

    logger.info("Secret files prepared")


###############################################################################
# DOCKER
###############################################################################


def docker_build(repo_dir, version, environment):
    """
    Build docker image.
    """

    image = (
        f"{REGISTRY}/"
        f"{PROJECT_NAME}:"
        f"{version}"
    )

    run([
        "docker",
        "build",
        "--build-arg",
        f"APP_VERSION={version}",
        "--build-arg",
        f"APP_ENV={environment}",
        "-t",
        image,
        "."
    ], cwd=repo_dir)

    return image


def docker_push(image):
    """
    Push image to registry.
    """

    run([
        "docker",
        "push",
        image
    ])


###############################################################################
# MAIN
###############################################################################


def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--env",
        required=True,
        choices=[
            "dev",
            "stage",
            "prod"
        ]
    )

    parser.add_argument(
        "--git-ref",
        required=True
    )

    return parser.parse_args()


def main():
    args = parse_args()

    logger.info(
        "Starting build env=%s ref=%s",
        args.env,
        args.git_ref
    )

    workdir = tempfile.mkdtemp()

    try:

        #
        # Clone repo
        #

        repo_dir = clone_repository(workdir)

        #
        # Checkout branch/tag
        #

        checkout_git_ref(
            repo_dir,
            args.git_ref
        )

        #
        # Read version
        #

        version = get_version(repo_dir)

        #
        # Load secrets
        #

        secrets = load_secrets(
            args.env
        )

        #
        # Generate configs
        #

        render_templates(
            repo_dir,
            secrets
        )

        #
        # Write files
        #

        write_secret_files(
            repo_dir,
            secrets
        )

        #
        # Build image
        #

        image = docker_build(
            repo_dir,
            version,
            args.env
        )

        #
        # Push image
        #

        docker_push(image)

        logger.info(
            "Build successful: %s",
            image
        )

    finally:
        shutil.rmtree(
            workdir,
            ignore_errors=True
        )


if __name__ == "__main__":
    try:
        main()
    except Exception as exc:
        logger.exception(exc)
        sys.exit(1)