Загрузка данных
#!/usr/bin/env python3
"""
Universal build script
Pipeline example:
python build.py \
--env prod \
--git-ref v1.2.3
Requirements:
pip install hvac jinja2
"""
import argparse
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import tomllib
import hvac
from jinja2 import Environment, FileSystemLoader
###############################################################################
# CONFIG
###############################################################################
PROJECT_NAME = "mailarchive"
REPO_URL = "https://dev.azure.com/company/project/_git/mailarchive"
REGISTRY = "registry.company.ru"
VAULT_PATHS = {
"dev": "projects/mailarchive/dev",
"stage": "projects/mailarchive/stage",
"prod": "projects/mailarchive/prod",
}
###############################################################################
# LOGGING
###############################################################################
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s"
)
logger = logging.getLogger(__name__)
###############################################################################
# HELPERS
###############################################################################
def run(cmd, cwd=None):
"""
Execute shell command.
Exit immediately on failure.
"""
logger.info("RUN: %s", " ".join(cmd))
subprocess.run(
cmd,
cwd=cwd,
check=True
)
###############################################################################
# GIT
###############################################################################
def clone_repository(workdir):
"""
Clone repository to temporary directory.
"""
repo_dir = os.path.join(workdir, PROJECT_NAME)
run([
"git",
"clone",
REPO_URL,
repo_dir
])
return repo_dir
def checkout_git_ref(repo_dir, git_ref):
"""
Checkout tag/branch/commit.
"""
run([
"git",
"checkout",
git_ref
], cwd=repo_dir)
###############################################################################
# VERSION
###############################################################################
def get_version(repo_dir):
"""
Read version from pyproject.toml
"""
pyproject = os.path.join(
repo_dir,
"pyproject.toml"
)
with open(pyproject, "rb") as f:
data = tomllib.load(f)
version = data["project"]["version"]
logger.info("Version: %s", version)
return version
###############################################################################
# VAULT
###############################################################################
def load_secrets(environment):
"""
Load secrets from Vault.
Required env vars:
VAULT_ADDR
VAULT_TOKEN
"""
logger.info(
"Loading secrets from Vault (%s)",
environment
)
client = hvac.Client(
url=os.environ["VAULT_ADDR"],
token=os.environ["VAULT_TOKEN"]
)
if not client.is_authenticated():
raise RuntimeError(
"Vault authentication failed"
)
path = VAULT_PATHS[environment]
response = (
client.secrets.kv.v2.read_secret_version(
path=path
)
)
secrets = response["data"]["data"]
logger.info(
"Loaded %s secret keys",
len(secrets)
)
return secrets
###############################################################################
# CONFIG GENERATION
###############################################################################
def render_templates(repo_dir, secrets):
"""
Render config templates.
Example:
templates/vhost.conf.j2
->
build/config/vhost.conf
"""
template_dir = os.path.join(
repo_dir,
"templates"
)
output_dir = os.path.join(
repo_dir,
"build",
"config"
)
os.makedirs(output_dir, exist_ok=True)
env = Environment(
loader=FileSystemLoader(template_dir)
)
#
# Example Apache vhost template
#
template = env.get_template(
"vhost.conf.j2"
)
rendered = template.render(
server_name=secrets.get("server_name"),
ldap_password=secrets.get("ldap_password"),
db_password=secrets.get("db_password"),
)
with open(
os.path.join(output_dir, "vhost.conf"),
"w"
) as f:
f.write(rendered)
logger.info("Templates rendered")
###############################################################################
# FILE SECRETS
###############################################################################
def write_secret_files(repo_dir, secrets):
"""
If Vault stores certificates, keytabs,
htpasswd etc.
"""
secrets_dir = os.path.join(
repo_dir,
"build",
"secrets"
)
os.makedirs(secrets_dir, exist_ok=True)
#
# Example:
# keytab stored in Vault as base64
#
# import base64
#
# with open(
# f"{secrets_dir}/service.keytab",
# "wb"
# ) as f:
# f.write(
# base64.b64decode(
# secrets["keytab"]
# )
# )
logger.info("Secret files prepared")
###############################################################################
# DOCKER
###############################################################################
def docker_build(repo_dir, version, environment):
"""
Build docker image.
"""
image = (
f"{REGISTRY}/"
f"{PROJECT_NAME}:"
f"{version}"
)
run([
"docker",
"build",
"--build-arg",
f"APP_VERSION={version}",
"--build-arg",
f"APP_ENV={environment}",
"-t",
image,
"."
], cwd=repo_dir)
return image
def docker_push(image):
"""
Push image to registry.
"""
run([
"docker",
"push",
image
])
###############################################################################
# MAIN
###############################################################################
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument(
"--env",
required=True,
choices=[
"dev",
"stage",
"prod"
]
)
parser.add_argument(
"--git-ref",
required=True
)
return parser.parse_args()
def main():
args = parse_args()
logger.info(
"Starting build env=%s ref=%s",
args.env,
args.git_ref
)
workdir = tempfile.mkdtemp()
try:
#
# Clone repo
#
repo_dir = clone_repository(workdir)
#
# Checkout branch/tag
#
checkout_git_ref(
repo_dir,
args.git_ref
)
#
# Read version
#
version = get_version(repo_dir)
#
# Load secrets
#
secrets = load_secrets(
args.env
)
#
# Generate configs
#
render_templates(
repo_dir,
secrets
)
#
# Write files
#
write_secret_files(
repo_dir,
secrets
)
#
# Build image
#
image = docker_build(
repo_dir,
version,
args.env
)
#
# Push image
#
docker_push(image)
logger.info(
"Build successful: %s",
image
)
finally:
shutil.rmtree(
workdir,
ignore_errors=True
)
if __name__ == "__main__":
try:
main()
except Exception as exc:
logger.exception(exc)
sys.exit(1)