From c7354c74e338a9fae83ec84a682f1e36d5d0f953 Mon Sep 17 00:00:00 2001 From: Geoff Wilson Date: Mon, 20 Oct 2025 12:19:34 -0400 Subject: [PATCH 1/5] Adding workflow for building docker images (PRs and release) --- .github/workflows/docker-build.yml | 95 +++++++++++++++++++++++++++ .github/workflows/release.yml | 100 +++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 .github/workflows/docker-build.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..2acdff6 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,95 @@ +name: Build and Push Docker Images + +on: + push: + branches: + - main + tags: + - 'v*.*.*' + - '[0-9]+.[0-9]+.[0-9]+' + pull_request: + branches: + - main + workflow_dispatch: + workflow_call: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + name: Build and Push Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify registry login + run: | + echo "✓ Successfully authenticated to ${{ env.REGISTRY }}" + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "ℹ️ PR build: Image will be built but not pushed to registry" + else + echo "ℹ️ Image will be built and pushed to registry" + fi + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # Tag with 'latest' on main branch + type=raw,value=latest,enable={{is_default_branch}} + # Tag with version on release tags (e.g., v1.2.3 -> 1.2.3) + type=semver,pattern={{version}} + # Tag with major.minor on release tags (e.g., v1.2.3 -> 1.2) + type=semver,pattern={{major}}.{{minor}} + # Tag with major on release tags (e.g., v1.2.3 -> 1) + type=semver,pattern={{major}} + # Tag with PR number on pull requests + type=ref,event=pr,prefix=pr- + # Tag with commit SHA for traceability + type=sha,prefix={{branch}}-,format=short + labels: | + org.opencontainers.image.title=Pulp Manager + org.opencontainers.image.description=FastAPI-based orchestration and management for multiple Pulp 3 servers + org.opencontainers.image.vendor=G-Research + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + - name: Generate artifact attestation + if: github.event_name != 'pull_request' + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.meta.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..be9b4e5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,100 @@ +name: Release + +on: + push: + tags: + - 'v*.*.*' + - '[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + inputs: + tag: + description: 'Tag to release' + required: true + type: string + +jobs: + build-docker: + name: Build Docker Image + uses: ./.github/workflows/docker-build.yml + permissions: + contents: read + packages: write + id-token: write + secrets: inherit + + create-release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: build-docker + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get version from tag + id: get_version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ inputs.tag }}" + else + VERSION=${GITHUB_REF#refs/tags/} + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Version: $VERSION" + + - name: Generate changelog + id: changelog + run: | + # Get the previous tag + PREV_TAG=$(git tag --sort=-v:refname | grep -E '^v?[0-9]+\.[0-9]+\.[0-9]+$' | sed -n '2p') + + if [ -z "$PREV_TAG" ]; then + echo "No previous tag found, using all commits" + PREV_TAG=$(git rev-list --max-parents=0 HEAD) + fi + + echo "## What's Changed" > changelog.md + echo "" >> changelog.md + + # Generate commit log + git log ${PREV_TAG}..HEAD --pretty=format:"* %s (%h)" --no-merges >> changelog.md + + echo "" >> changelog.md + echo "" >> changelog.md + echo "**Full Changelog**: https://github.com/${{ github.repository }}/compare/${PREV_TAG}...${{ steps.get_version.outputs.version }}" >> changelog.md + + cat changelog.md + + - name: Create Release + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const changelog = fs.readFileSync('changelog.md', 'utf8'); + + const release = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: '${{ steps.get_version.outputs.version }}', + name: 'Release ${{ steps.get_version.outputs.version }}', + body: changelog, + draft: false, + prerelease: false, + make_latest: 'true' + }); + + console.log(`Created release ${release.data.html_url}`); + + // Add Docker image information to release + const dockerImage = `ghcr.io/${{ github.repository }}:${{ steps.get_version.outputs.version }}`; + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release.data.id, + body: changelog + `\n\n## Docker Image\n\n\`\`\`bash\ndocker pull ${dockerImage}\n\`\`\`` + }); From 1f420dcf908f3835b7f12dd2038c21cbd84de039 Mon Sep 17 00:00:00 2001 From: Geoff Wilson Date: Mon, 20 Oct 2025 12:45:30 -0400 Subject: [PATCH 2/5] Fix PR build Signed-off-by: Geoff Wilson --- .github/workflows/docker-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 2acdff6..d248bff 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -67,8 +67,10 @@ jobs: type=semver,pattern={{major}} # Tag with PR number on pull requests type=ref,event=pr,prefix=pr- + # Tag with branch name on branch pushes + type=ref,event=branch # Tag with commit SHA for traceability - type=sha,prefix={{branch}}-,format=short + type=sha,format=short labels: | org.opencontainers.image.title=Pulp Manager org.opencontainers.image.description=FastAPI-based orchestration and management for multiple Pulp 3 servers From 773aff189cc7b8a8b8556eae960c94379ed594cc Mon Sep 17 00:00:00 2001 From: Geoff Wilson Date: Mon, 20 Oct 2025 14:10:35 -0400 Subject: [PATCH 3/5] Simplify push to registry logic Signed-off-by: Geoff Wilson --- .github/workflows/docker-build.yml | 121 +++++++++++++++++++---------- 1 file changed, 82 insertions(+), 39 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d248bff..5472653 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -14,8 +14,7 @@ on: workflow_call: env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: pulp-manager jobs: build-and-push: @@ -35,63 +34,107 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Log in to Container Registry + - name: Determine registries to push to + id: registries + run: | + # Always include ghcr.io + registries="ghcr.io" + + # Add docker.io and quay.io for release tags + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + registries="ghcr.io docker.io quay.io" + fi + + echo "registries=${registries}" >> $GITHUB_OUTPUT + echo "Will push to: ${registries}" + + - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Verify registry login + - name: Log in to Docker Hub + if: startsWith(github.ref, 'refs/tags/') + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKER_BOT_USERNAME }} + password: ${{ secrets.DOCKER_BOT_PASSWORD }} + + - name: Log in to Quay.io + if: startsWith(github.ref, 'refs/tags/') + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_BOT_USERNAME }} + password: ${{ secrets.QUAY_BOT_PASSWORD }} + + - name: Determine tags + id: tags run: | - echo "✓ Successfully authenticated to ${{ env.REGISTRY }}" - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "ℹ️ PR build: Image will be built but not pushed to registry" + tags="" + + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + # Release tag (e.g., v1.2.3) + version="${{ github.ref_name }}" + version="${version#v}" # Remove 'v' prefix + major=$(echo $version | cut -d. -f1) + minor=$(echo $version | cut -d. -f1-2) + tags="${version} ${minor} ${major} latest" + elif [ "${{ github.ref_name }}" == "main" ]; then + # Main branch + tags="main latest" + elif [ "${{ github.event_name }}" == "pull_request" ]; then + # PR + tags="pr-${{ github.event.pull_request.number }}" else - echo "ℹ️ Image will be built and pushed to registry" + # Other branches + tags="${{ github.ref_name }}" fi - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - # Tag with 'latest' on main branch - type=raw,value=latest,enable={{is_default_branch}} - # Tag with version on release tags (e.g., v1.2.3 -> 1.2.3) - type=semver,pattern={{version}} - # Tag with major.minor on release tags (e.g., v1.2.3 -> 1.2) - type=semver,pattern={{major}}.{{minor}} - # Tag with major on release tags (e.g., v1.2.3 -> 1) - type=semver,pattern={{major}} - # Tag with PR number on pull requests - type=ref,event=pr,prefix=pr- - # Tag with branch name on branch pushes - type=ref,event=branch - # Tag with commit SHA for traceability - type=sha,format=short - labels: | - org.opencontainers.image.title=Pulp Manager - org.opencontainers.image.description=FastAPI-based orchestration and management for multiple Pulp 3 servers - org.opencontainers.image.vendor=G-Research + # Add SHA tag for traceability + sha_short=$(echo ${{ github.sha }} | cut -c1-7) + tags="${tags} sha-${sha_short}" + + echo "tags=${tags}" >> $GITHUB_OUTPUT + echo "Will use tags: ${tags}" - - name: Build and push Docker image + - name: Build Docker image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + push: false + load: true + tags: pulp/${{ env.IMAGE_NAME }}:ci + labels: | + org.opencontainers.image.title=Pulp Manager + org.opencontainers.image.description=FastAPI-based orchestration and management for multiple Pulp 3 servers + org.opencontainers.image.vendor=Pulp + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64 + - name: Push to registries + if: github.event_name != 'pull_request' + run: | + for registry in ${{ steps.registries.outputs.registries }}; do + echo "Pushing to ${registry}..." + for tag in ${{ steps.tags.outputs.tags }}; do + echo " Tagging and pushing ${registry}/pulp/${{ env.IMAGE_NAME }}:${tag}" + docker tag pulp/${{ env.IMAGE_NAME }}:ci ${registry}/pulp/${{ env.IMAGE_NAME }}:${tag} + docker push ${registry}/pulp/${{ env.IMAGE_NAME }}:${tag} + done + done + - name: Generate artifact attestation if: github.event_name != 'pull_request' uses: actions/attest-build-provenance@v1 with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.meta.outputs.digest }} + subject-name: ghcr.io/pulp/${{ env.IMAGE_NAME }} + subject-digest: ${{ hashFiles('Dockerfile') }} push-to-registry: true From aae586925ec940a9052961ceb8378dfd36874e11 Mon Sep 17 00:00:00 2001 From: Geoff Wilson Date: Thu, 23 Oct 2025 10:45:15 -0400 Subject: [PATCH 4/5] Improve the dockerfile product Signed-off-by: Geoff Wilson --- .dockerignore | 45 +++++++++++++++++++-- Dockerfile | 20 +++++++++- config-examples/README.md | 70 +++++++++++++++++++++++++++++++++ config-examples/config.ini | 55 ++++++++++++++++++++++++++ config-examples/pulp_config.yml | 44 +++++++++++++++++++++ 5 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 config-examples/README.md create mode 100644 config-examples/config.ini create mode 100644 config-examples/pulp_config.yml diff --git a/.dockerignore b/.dockerignore index 6111977..6e209a3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,43 @@ -demo/assets/keys/gpg/ -demo/assets/certs/ +# Version control +.git/ +.gitignore +.github/ + +# Development environment +.devcontainer/ .claude/ -.coverage venv/ -__pycache__/ \ No newline at end of file + +# Tests and coverage +**/tests/ +**/unit_tests/ +.pytest_cache/ +.coverage +htmlcov/ +pytest.ini + +# Development configs +pylint.rc +Makefile + +# Documentation +*.md +!README.md +docs/ + +# Demo files +demo/ + +# Python cache +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +*.egg-info/ + +# IDE files +.vscode/ +.idea/ +*.swp +.DS_Store diff --git a/Dockerfile b/Dockerfile index 4e50084..c6b7060 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,11 +39,27 @@ RUN apt-get update && apt-get install -y netcat-openbsd git make python3-dev lib COPY --from=builder /opt/venv /opt/venv # Copy requirements file COPY --from=builder /pulp_manager/requirements.txt ./ -# Copy the entire project -COPY . . + +# Copy application code +COPY pulp_manager ./pulp_manager/ +COPY pulp3_bindings ./pulp3_bindings/ +COPY hashi_vault_client ./hashi_vault_client/ + +# Copy database migration files +COPY alembic ./alembic/ +COPY alembic.ini ./ + +# Copy entrypoint script +COPY pulp-manager.sh ./ + +# Install default configs to /etc/pulp_manager/ +RUN mkdir -p /etc/pulp_manager +COPY config-examples/config.ini /etc/pulp_manager/config.ini +COPY config-examples/pulp_config.yml /etc/pulp_manager/pulp_config.yml # Ensure correct permissions RUN chown -R pulp_manager:pulp_manager /pulp_manager \ + && chown -R pulp_manager:pulp_manager /etc/pulp_manager \ && ln -s /pulp_manager/pulp-manager.sh /usr/local/bin/pulp-manager USER 10001 diff --git a/config-examples/README.md b/config-examples/README.md new file mode 100644 index 0000000..96426be --- /dev/null +++ b/config-examples/README.md @@ -0,0 +1,70 @@ +# Configuration Examples + +This directory contains example configuration files that are copied into the Docker image at `/etc/pulp_manager/`. + +## Files + +- `config.ini` - Main application configuration +- `pulp_config.yml` - Pulp server and sync schedule configuration + +## Usage + +### Docker Image Default Configs + +These configs are automatically copied to `/etc/pulp_manager/` during image build and serve as working defaults for testing and CI. + +### Production Deployment + +For production use, you have two options: + +#### Option 1: Mount custom configs at the default location + +```bash +docker run -v /path/to/your/config.ini:/etc/pulp_manager/config.ini \ + -v /path/to/your/pulp_config.yml:/etc/pulp_manager/pulp_config.yml \ + pulp/pulp-manager +``` + +#### Option 2: Mount configs anywhere and use environment variables + +```bash +docker run -v /path/to/configs:/configs \ + -e PULP_MANAGER_CONFIG_PATH=/configs/my-config.ini \ + -e PULP_SYNC_CONFIG_PATH=/configs/my-pulp-config.yml \ + pulp/pulp-manager +``` + +### Environment Variable Overrides + +The default `config.ini` supports environment variable substitution for common settings: + +- `DB_HOSTNAME` - Database host (default: localhost) +- `DB_PORT` - Database port (default: 3306) +- `DB_NAME` - Database name (default: pulp_manager) +- `DB_USER` - Database user (default: pulp-manager) +- `DB_PASSWORD` - Database password (default: pulp-manager) +- `REDIS_HOST` - Redis host (default: localhost) +- `REDIS_PORT` - Redis port (default: 6379) +- `REDIS_DB` - Redis database number (default: 0) +- `PULP_ADMIN_PASSWORD` - Pulp admin password (default: password) + +Example using environment variables: + +```bash +docker run -e DB_HOSTNAME=mysql.example.com \ + -e DB_PASSWORD=secret \ + -e REDIS_HOST=redis.example.com \ + pulp/pulp-manager +``` + +## Configuration Locations + +The application looks for configs in this order: + +1. Environment variable `PULP_MANAGER_CONFIG_PATH` (if set) +2. Default location `/etc/pulp_manager/config.ini` + +And for Pulp sync config: + +1. Environment variable `PULP_SYNC_CONFIG_PATH` (if set) +2. Default location `/etc/pulp_manager/pulp_config.yml` diff --git a/config-examples/config.ini b/config-examples/config.ini new file mode 100644 index 0000000..300316d --- /dev/null +++ b/config-examples/config.ini @@ -0,0 +1,55 @@ +# Pulp Manager Configuration +# This is an example configuration file included in the Docker image. +# For production use, mount your own config.ini at /etc/pulp_manager/config.ini +# or set PULP_MANAGER_CONFIG_PATH environment variable. + +[database] +# Database connection settings +# Override with environment variables: DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD +user=${DB_USER:pulp-manager} +password=${DB_PASSWORD:pulp-manager} +host=${DB_HOSTNAME:localhost} +port=${DB_PORT:3306} +db_name=${DB_NAME:pulp_manager} + +[auth] +# Authentication method: 'ldap' or 'none' (for testing only) +method=none +use_ssl=false +# LDAP settings (if method=ldap) +ldap_servers= +base_dn= +default_domain= +jwt_algorithm=HS256 +jwt_token_lifetime_mins=480 +admin_group= +# For production, set to true +require_jwt_auth=false + +[pulp] +# Pulp server configuration +# Comma-separated list of internal Pulp domains +internal_domains= +# Git repository config directory (optional) +git_repo_config_dir= +# Pulp admin password +password=${PULP_ADMIN_PASSWORD:password} +# Package filtering (optional) +banned_package_regex= +internal_package_prefix= +package_name_replacement_pattern= +# Signing service (optional) +# deb_signing_service= + +[redis] +# Redis connection for background workers +host=${REDIS_HOST:localhost} +port=${REDIS_PORT:6379} +db=${REDIS_DB:0} + +[vault] +# HashiCorp Vault settings (optional) +# enabled=false +# url= +# role= +# mount_point= diff --git a/config-examples/pulp_config.yml b/config-examples/pulp_config.yml new file mode 100644 index 0000000..e16b741 --- /dev/null +++ b/config-examples/pulp_config.yml @@ -0,0 +1,44 @@ +# Pulp Manager - Pulp Server Configuration +# This is an example configuration file included in the Docker image. +# For production use, mount your own pulp_config.yml at /etc/pulp_manager/pulp_config.yml +# or set PULP_SYNC_CONFIG_PATH environment variable. + +# Example configuration with no servers defined +# Add your Pulp servers and sync schedules here + +pulp_servers: {} +# Example server configuration: +# pulp_servers: +# pulp-primary:80: +# credentials: local +# repo_groups: +# default: +# schedule: "0 * * * *" # Every hour +# max_concurrent_syncs: 1 +# max_runtime: "1h" +# +# pulp-secondary:80: +# credentials: local +# repo_groups: +# external_repos: +# schedule: "*/5 * * * *" # Every 5 minutes +# max_concurrent_syncs: 2 +# max_runtime: "6h" +# pulp_master: pulp-primary:80 + +credentials: {} +# Example credentials: +# credentials: +# local: +# username: admin +# vault_service_account_mount: local-auth + +repo_groups: {} +# Example repo groups: +# repo_groups: +# default: +# regex_include: ".*" # Match all repositories +# external_repos: +# regex_include: "^ext" # Match repositories starting with "ext" +# internal_repos: +# regex_include: "^int" # Match repositories starting with "int" From 4f01756927423f8d0256283937af9b4fd4736aa7 Mon Sep 17 00:00:00 2001 From: Geoff Wilson Date: Thu, 23 Oct 2025 13:57:53 -0400 Subject: [PATCH 5/5] Example config vals Signed-off-by: Geoff Wilson --- config-examples/config.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config-examples/config.ini b/config-examples/config.ini index 300316d..143611f 100644 --- a/config-examples/config.ini +++ b/config-examples/config.ini @@ -32,12 +32,13 @@ require_jwt_auth=false internal_domains= # Git repository config directory (optional) git_repo_config_dir= -# Pulp admin password +# Pulp admin password for use in dev/test only password=${PULP_ADMIN_PASSWORD:password} # Package filtering (optional) -banned_package_regex= -internal_package_prefix= -package_name_replacement_pattern= +banned_package_regex=blocked_package|another +package_name_replacement_pattern=^(?P[a-z]+)-(?P[a-z]+)-(?P.+)$ +package_name_replacement_rule={env}_{org}_{pkg} +internal_package_prefix=int_ # Signing service (optional) # deb_signing_service=