From fb0e4d705bbe51d405131048f304b26608788a87 Mon Sep 17 00:00:00 2001 From: Michael Housh Date: Mon, 15 Dec 2025 12:21:13 -0500 Subject: [PATCH] feat: Adds airflow assessment generation to hpa scripts. --- env/.config/aap/config.toml | 75 ++++++++++++++++++++++++ env/.local/scripts/hpa | 2 + env/.local/scripts/hpa-create | 5 +- env/.local/scripts/hpa-init | 61 ++++++++++---------- env/.local/scripts/hpa-pull | 76 +++++++++++++------------ env/.local/scripts/utils/hpa/aap | 47 +++++++++++++++ env/.local/scripts/utils/hpa/hpa-create | 32 ++++++++--- env/.local/scripts/utils/hpa/hpa.env | 8 +++ 8 files changed, 230 insertions(+), 76 deletions(-) create mode 100644 env/.config/aap/config.toml create mode 100755 env/.local/scripts/utils/hpa/aap diff --git a/env/.config/aap/config.toml b/env/.config/aap/config.toml new file mode 100644 index 0000000..0e8676c --- /dev/null +++ b/env/.config/aap/config.toml @@ -0,0 +1,75 @@ +# This config is setup for running inside the docker container. +# +# NOTE: +# Configuration settings for the `hpa` command line tool. +# You can delete settings that are not applicable to your use case. + +# Default arguments / options that get passed into `ansible-playbook` commands. +# WARNING: Do not put arguments / options that contain spaces in the same string, +# they should be separate strings, for example do not do something like +# ['--tags debug'], instead use ['--tags', 'debug']. +# +args = ['--tags', 'debug'] + +# Set to true if you want to pass the vault args to `ansible-playbook` commands. +useVaultArgs = true + +# NOTE: +# Configuration for running the generate command(s). This allows custimizations +# to the files that get used to generate the final output (generally a pdf). +# See `pandoc --help`. Below are the defaults that get used, which only need +# adjusted if your template does not follow the default template design or if +# you add extra files to your template that need to be included in the final +# output. Also be aware that any of the files specified in the `files` or +# `includeInHeader` options, need to be inside the `buildDirectory` when generating +# the final output file. + +# [generate] +# this relative to the project directory. +# buildDirectory = '.build' +# pdfEngine = 'xelatex' +# includeInHeader = [ +# 'head.tex', +# 'footer.tex' +# ] +# files = [ +# 'Report.md', +# 'Definitions.md' +# ] +# outputFileName = 'Report' + +# NOTE: +# These are more for local development of the ansible playbook and should not be needed +# in most cases. Uncomment the lines if you want to customize the playbook and use it +# instead of the provided / default playbook. + +[playbook] +directory = '/root/.local/share/hpa/playbook' +inventory = '/root/.local/share/hpa/playbook/inventory.ini' +version = 'main' + +# NOTE: +# These are to declare where your template files are either on your local system or +# a remote git repository. +[template] +# The directory path on your local system to the template files. +directory = '/root/.local/share/hpa/airflow-assessment-template' +vars = 'repo_vars' + +# The url to a git repository that contains your template files. +# url = 'https://git.example.com/consult-template.git' + +# The version, tag, branch, or sha of the template files to clone from the remote +# template repository. In general it is best practice to use a version instead of a +# branch. +# version = '1.0.0' + +# NOTE: +# Holds settings for `ansible-vault` commands. +[vault] +# Arguments to pass to commands that use `ansible-vault`, such as encrypting or decrypting +# files. +args = ['--vault-password-file=/run/secrets/vault-pass'] + +# An id to use when encrypting `ansible-vault` files. +#encryptId = 'myId' diff --git a/env/.local/scripts/hpa b/env/.local/scripts/hpa index b6d6cc7..bfc82b3 100755 --- a/env/.local/scripts/hpa +++ b/env/.local/scripts/hpa @@ -38,6 +38,8 @@ if echo "$HPA_DOCKER_IMAGE" | grep -vq "localhost"; then fi fi +echo "ARGS: $@" + podman run --rm --interactive --tty \ --volume "$HPA_CONFIG_DIR":/root/.config/hpa:ro \ --volume "$HPA_DATA_DIR":/root/.local/share/hpa:ro \ diff --git a/env/.local/scripts/hpa-create b/env/.local/scripts/hpa-create index 475034b..0e8e98e 100755 --- a/env/.local/scripts/hpa-create +++ b/env/.local/scripts/hpa-create @@ -4,7 +4,8 @@ title="Creating project..." script="${SCRIPTS:-$HOME/.local/scripts}/utils/hpa/hpa-create" first_arg=${1:-""} if [[ $first_arg == "-h" ]] || [[ $first_arg == "--help" ]]; then - . "$script" $* + . "$script" $* else - gum spin --show-output --title "$title" -- bash -c "$script $*" | tr -d '\r' | head -1 + # bash -c "$script $*" | tr -d '\r' | head -1 + gum spin --show-output --title "$title" -- bash -c "$script $*" | tr -d '\r' | head -1 fi diff --git a/env/.local/scripts/hpa-init b/env/.local/scripts/hpa-init index 78629ef..298d051 100755 --- a/env/.local/scripts/hpa-init +++ b/env/.local/scripts/hpa-init @@ -14,7 +14,7 @@ LOG_FILE=${LOG_FILE:-"$LOG_LABEL.log"} . "$SCRIPTS/utils/hpa/hpa.env" usage() { - cat <"$output" + [[ ! -d "$output_dir" ]] && mkdir -p "$output_dir" + ( + podman run --rm -it "$HPA_DOCKER_IMAGE:$HPA_DOCKER_TAG" \ + --generate-completion-script "$type" | + tr -d '\r' + ) >"$output" } generate-secret() { - log "Generating vault secret for key: '$HPA_VAULT_SECRET_KEY'" - local secret - secret="$(pass -c ansible/vault-pass)" - printf "%s" "$secret" | podman secret create "$HPA_VAULT_SECRET_KEY" - + log "Generating vault secret for key: '$HPA_VAULT_SECRET_KEY'" + local secret + secret="$(pass -c ansible/vault-pass)" + printf "%s" "$secret" | podman secret create "$HPA_VAULT_SECRET_KEY" - } ################################################################################ @@ -76,22 +76,23 @@ generate-secret() { first_arg=${1:-""} if [[ $first_arg == "-h" ]] || [[ $first_arg == "--help" ]]; then - usage && exit 0 + usage && exit 0 else - # Setup logging file and label. - source "$SCRIPTS/hypr/logging" - setup-logging "$LOG_FILE" "$LOG_LABEL" + # Setup logging file and label. + source "$SCRIPTS/hypr/logging" + setup-logging "$LOG_FILE" "$LOG_LABEL" - log "Starting init..." + log "Starting init..." - generate-completion "$@" - generate-secret + generate-completion "$@" + generate-secret - log "Generating directories, if they don't exist." - mkdir -p "$HPA_CONSULTS_DIR" &>/dev/null - mkdir "$HPA_DATA_DIR" &>/dev/null + log "Generating directories, if they don't exist." + mkdir -p "$HPA_CONSULTS_DIR" &>/dev/null + mkdir "$HPA_DATA_DIR" &>/dev/null - log "Cloning required template and playbook, if they don't exist" - [[ ! -d "$HPA_PLAYBOOK_DIR" ]] && git clone "$HPA_PLAYBOOK_URL" "$HPA_PLAYBOOK_DIR" - [[ ! -d "$HPA_CONSULT_TEMPLATE_DIR" ]] && git clone "$HPA_CONSULT_TEMPLATE_URL" "$HPA_CONSULT_TEMPLATE_DIR" + log "Cloning required template and playbook, if they don't exist" + [[ ! -d "$HPA_PLAYBOOK_DIR" ]] && git clone "$HPA_PLAYBOOK_URL" "$HPA_PLAYBOOK_DIR" + [[ ! -d "$HPA_CONSULT_TEMPLATE_DIR" ]] && git clone "$HPA_CONSULT_TEMPLATE_URL" "$HPA_CONSULT_TEMPLATE_DIR" + [[ ! -d "$HPA_AIRFLOW_TEMPLATE_DIR" ]] && git clone "$HPA_AIRFLOW_TEMPLATE_URL" "$HPA_AIRFLOW_TEMPLATE_DIR" fi diff --git a/env/.local/scripts/hpa-pull b/env/.local/scripts/hpa-pull index c23f33b..64c517f 100755 --- a/env/.local/scripts/hpa-pull +++ b/env/.local/scripts/hpa-pull @@ -13,7 +13,7 @@ LOG_FILE=${LOG_FILE:-"$LOG_LABEL.log"} . "$SCRIPTS/utils/hpa/hpa.env" usage() { - cat </dev/null || exit 1 - ( - git pull - ) - popd &>/dev/null + pushd "$dir" &>/dev/null || exit 1 + ( + git pull + ) + popd &>/dev/null } pull-docker() { - log --echo "Pulling docker image: '$HPA_DOCKER_IMAGE:$HPA_DOCKER_TAG'" - podman pull "$HPA_DOCKER_IMAGE:$HPA_DOCKER_TAG" + log --echo "Pulling docker image: '$HPA_DOCKER_IMAGE:$HPA_DOCKER_TAG'" + podman pull "$HPA_DOCKER_IMAGE:$HPA_DOCKER_TAG" } pull-playbook() { - log --echo "Pulling playbook: '$HPA_PLAYBOOK_DIR'" - pull-repo "$HPA_PLAYBOOK_DIR" + log --echo "Pulling playbook: '$HPA_PLAYBOOK_DIR'" + pull-repo "$HPA_PLAYBOOK_DIR" } pull-template() { - log --echo "Pulling template: '$HPA_CONSULT_TEMPLATE_DIR'" - pull-repo "$HPA_CONSULT_TEMPLATE_DIR" + log --echo "Pulling template: '$HPA_CONSULT_TEMPLATE_DIR'" + pull-repo "$HPA_CONSULT_TEMPLATE_DIR" + + log --echo "Pulling airflow assessment template: '$HPA_AIRFLOW_TEMPLATE_DIR'" + pull-repo "$HPA_AIRFLOW_TEMPLATE_DIR" } ################################################################################ @@ -78,31 +81,32 @@ playbook_flag="0" template_flag="0" while [[ $# -gt 0 ]]; do - if [[ $1 == "-a" ]] || [[ $1 == "--all" ]]; then - all_flag="1" - break - elif [[ $1 == "-h" ]] || [[ $1 == "--help" ]]; then - usage && exit 0 - elif [[ $1 == "-d" ]] || [[ $1 == "--docker" ]]; then - all_flag="0" - docker_flag="1" - elif [[ $1 == "-p" ]] || [[ $1 == "--playbook" ]]; then - all_flag="0" - playbook_flag="1" - elif [[ $1 == "-t" ]] || [[ $1 == "--template" ]]; then - all_flag="0" - template_flag="1" - fi - shift + if [[ $1 == "-a" ]] || [[ $1 == "--all" ]]; then + all_flag="1" + break + elif [[ $1 == "-h" ]] || [[ $1 == "--help" ]]; then + usage && exit 0 + elif [[ $1 == "-d" ]] || [[ $1 == "--docker" ]]; then + all_flag="0" + docker_flag="1" + elif [[ $1 == "-p" ]] || [[ $1 == "--playbook" ]]; then + all_flag="0" + playbook_flag="1" + elif [[ $1 == "-t" ]] || [[ $1 == "--template" ]]; then + all_flag="0" + template_flag="1" + fi + shift done if [[ $all_flag == "1" ]]; then - docker_flag="1" - playbook_flag="1" - template_flag="1" + docker_flag="1" + playbook_flag="1" + template_flag="1" fi [[ $docker_flag == "1" ]] && pull-docker [[ $playbook_flag == "1" ]] && pull-playbook [[ $template_flag == "1" ]] && pull-template date +%s >"$HPA_CONFIG_DIR/.lastpull" +date +%s >"$HPA_AIRFLOW_CONFIG_DIR/.lastpull" diff --git a/env/.local/scripts/utils/hpa/aap b/env/.local/scripts/utils/hpa/aap new file mode 100755 index 0000000..700e804 --- /dev/null +++ b/env/.local/scripts/utils/hpa/aap @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -e +set -o nounset +set -o pipefail + +# A wrapper script to run swift-hpa in a docker container and +# mount the correct volumes, etc. +# +# Make sure to run 'hpa-init' first on this machine to setup +# dependencies, if you have not done so already. +# +# To attach to a shell inside the container run: +# `hpa bash` +# +# To run ansible vault commands inside the container run: +# `hpa ansible-vault ...` +# + +SCRIPTS=${SCRIPTS:-$HOME/.local/scripts} +. "$SCRIPTS/utils/hpa/hpa.env" + +############################## MAIN ############################## + +# Don't pull images if they're prefixed with 'localhost'. +if echo "$HPA_DOCKER_IMAGE" | grep -vq "localhost"; then + + # Check if we should pull the image prior to running. + last_pull="0" + if [[ -f "$HPA_AIRFLOW_CONFIG_DIR/.lastpull" ]]; then + last_pull=$(cat "$HPA_AIRFLOW_CONFIG_DIR/.lastpull") + fi + curr=$(date +%s) + diff=$((curr - last_pull)) + + if [[ $HPA_AUTO_PULL == "1" ]] && [[ $diff -gt "$HPA_AUTO_PULL_INTERVAL" ]]; then + . "$SCRIPTS/hpa-pull" "$HPA_AUTO_PULL_OPTS" + fi +fi + +podman run --rm --interactive --tty \ + --volume "$HPA_AIRFLOW_CONFIG_DIR":/root/.config/hpa:ro \ + --volume "$HPA_DATA_DIR":/root/.local/share/hpa:ro \ + --volume "$HPA_CONSULTS_DIR":/consults \ + --volume "$PWD":/root/project \ + --secret "$HPA_VAULT_SECRET_KEY" \ + "$HPA_DOCKER_IMAGE:$HPA_DOCKER_TAG" "$@" diff --git a/env/.local/scripts/utils/hpa/hpa-create b/env/.local/scripts/utils/hpa/hpa-create index b5e6c1f..7422bc6 100755 --- a/env/.local/scripts/utils/hpa/hpa-create +++ b/env/.local/scripts/utils/hpa/hpa-create @@ -17,7 +17,8 @@ LOG_FILE=${LOG_FILE:-"$LOG_LABEL.log"} # Load environment / shared variables. . "$SCRIPTS/utils/hpa/hpa.env" -declare no_git_flag no_push_flag date_opt +declare airflow_flag no_git_flag no_push_flag date_opt +airflow_flag="0" no_git_flag="0" no_push_flag="0" date_opt=$(date '+%Y.%m.%d') @@ -25,7 +26,8 @@ date_opt=$(date '+%Y.%m.%d') usage() { cat < FLAGS: + -a | --airflow: Generate using the airflow assessment project template. -d | --date: Override the date the project was started. -g | --no-git: Do not initialize a git repository for the project. -h | --help: Show this help page. @@ -62,21 +65,32 @@ log() { } create() { - local customer container_dir + local customer container_dir script suffix customer=${1:-""} [[ -z $customer ]] && log --error "Must supply a customer name for the project" && exit 1 - [[ ! -f $SCRIPTS/hpa ]] && - log --error "Unable to find the 'hpa' script." && - exit 1 - log "Generating project for: '$customer'" + # Setup approriate configuration and project suffix. + if [[ $airflow_flag == "1" ]]; then + suffix="AAP" + script="$SCRIPTS/utils/hpa/aap" + else + suffix="HPA" + script="$SCRIPTS/hpa" + fi + + [[ ! -f $SCRIPTS/hpa ]] && + log --error "Unable to find the script: '$script'." && + exit 1 + + log "Using script: '$script'" + container_dir=$( - "$SCRIPTS/hpa" create --quiet "/consults/$date_opt.$customer" + "$script" create --quiet "/consults/$date_opt.$customer.$suffix" ) echo "$HPA_CONSULTS_DIR/$(basename "$container_dir")" } @@ -129,6 +143,8 @@ declare customer output while [[ $# -gt 0 ]]; do if [[ $1 == "-h" ]] || [[ $1 == "--help" ]]; then usage && exit 0 + elif [[ $1 == "-a" ]] || [[ $1 == "--airflow" ]]; then + airflow_flag="1" elif [[ $1 == "-d" ]] || [[ $1 == "--date" ]]; then shift date_opt="$1" diff --git a/env/.local/scripts/utils/hpa/hpa.env b/env/.local/scripts/utils/hpa/hpa.env index 91c28e8..bfc3431 100755 --- a/env/.local/scripts/utils/hpa/hpa.env +++ b/env/.local/scripts/utils/hpa/hpa.env @@ -33,6 +33,11 @@ HPA_CONSULT_ORIGIN_BASE_URL=${CONSULT_ORIGIN_BASE_URL:-"ssh://git@git.housh.dev: HPA_CONSULT_TEMPLATE_DIR=${HPA_DATA_DIR}/template HPA_CONSULT_TEMPLATE_URL="ssh://git@git.housh.dev:2222/hhe/consult-template.git" +# Airflow assessment vars. +HPA_AIRFLOW_CONFIG_DIR="${XDG_CONFIG_HOME}/aap" +HPA_AIRFLOW_TEMPLATE_DIR="${HPA_DATA_DIR}/airflow-assessment-template" +HPA_AIRFLOW_TEMPLATE_URL="ssh://git@git.housh.dev:2222/hhe/airflow-assessment-template.git" + export HPA_AUTO_PULL export HPA_AUTO_PULL_INTERVAL export HPA_AUTO_PULL_OPTS @@ -47,3 +52,6 @@ export HPA_DOCKER_TAG export HPA_PLAYBOOK_URL export HPA_PLAYBOOK_DIR export HPA_VAULT_SECRET_KEY +export HPA_AIRFLOW_CONFIG_DIR +export HPA_AIRFLOW_TEMPLATE_DIR +export HPA_AIRFLOW_TEMPLATE_URL