From 43f71bbb55402ef506ece990656ab6a17fcddb59 Mon Sep 17 00:00:00 2001 From: vickytechkey Date: Thu, 12 Nov 2026 14:08:59 +0530 Subject: [PATCH] ci(deploy): enforce integration-tests dependency gate and auto-rollback in beta workflow --- .forgejo/workflows/beta.yml | 92 +++++++++++++++++++++- .forgejo/workflows/integration-testing.yml | 65 --------------- 2 files changed, 90 insertions(+), 67 deletions(-) delete mode 100644 .forgejo/workflows/integration-testing.yml diff --git a/.forgejo/workflows/beta.yml b/.forgejo/workflows/beta.yml index 31c6e90..35b7c92 100644 --- a/.forgejo/workflows/beta.yml +++ b/.forgejo/workflows/beta.yml @@ -6,7 +6,61 @@ on: - beta jobs: + integration-tests: + name: Port Collision Audit & Inter-Service API Integration Testing + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: https://github.com/actions/checkout@v4 + + - name: Setup Python + run: | + if ! which python3 >/dev/null 2>&1; then + if which apt-get >/dev/null 2>&1; then + (apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-dev) || (sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv libpq-dev) + elif which apk >/dev/null 2>&1; then + apk add --no-cache python3 py3-pip + fi + fi + python3 --version + VENV_PATH="$HOME/venv" + python3 -m venv "$VENV_PATH" || (sudo apt-get update && sudo apt-get install -y python3-venv libpq-dev && python3 -m venv "$VENV_PATH") || true + if [ -d "$VENV_PATH/bin" ]; then + echo "$VENV_PATH/bin" >> $GITHUB_PATH + fi + + - name: Install Test Dependencies + run: | + if [ -f "$HOME/venv/bin/pip" ]; then + "$HOME/venv/bin/pip" install --upgrade pip + "$HOME/venv/bin/pip" install pytest requests psycopg2-binary + else + pip3 install --upgrade pip --break-system-packages || pip install --upgrade pip || true + pip3 install pytest requests psycopg2-binary --break-system-packages || pip install pytest requests psycopg2-binary + fi + + - name: Gate 1 - Port Collision Integrity Check + env: + PORT_DB_HOST: "172.31.15.84" + PORT_DB_USER: "vignesh" + PORT_DB_PASSWORD: "vignesh_secret_pass_2026" + run: | + echo "๐Ÿ”Ž Running Port Collision Audit against PostgreSQL portmanagement.porttable..." + python tools/port_integrity_check.py --service customer-website-backend --port 8082 || python3 tools/port_integrity_check.py --service customer-website-backend --port 8082 + + - name: Gate 2 - Inter-Service API Integration Testing + env: + EC2_HOST: "16.113.57.127" + DB_HOST: "172.31.15.84" + DB_USER: "vignesh" + DB_PASSWORD: "vignesh_secret_pass_2026" + run: | + echo "๐Ÿงช Running full inter-service integration test suite (Uploads, Couriers, CRM, KYC)..." + pytest tests/integration/test_inter_service_mesh.py -v || python3 -m pytest tests/integration/test_inter_service_mesh.py -v + deploy: + name: Deploy to Beta EC2 + needs: [integration-tests] runs-on: ubuntu-latest steps: - name: Checkout Code @@ -18,7 +72,8 @@ jobs: - name: Deploy to Beta EC2 run: | - apk add --no-cache openssh-client aws-cli + which apk >/dev/null 2>&1 && apk add --no-cache openssh-client aws-cli || true + which apt-get >/dev/null 2>&1 && (apt-get update && apt-get install -y openssh-client) || true mkdir -p ~/.ssh echo "${{ secrets.BETA_EC2_SSH_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa @@ -34,9 +89,16 @@ jobs: sudo apt-get update sudo apt-get install -y python3-pip python3-venv python3-dev libpq-dev build-essential - # โ”€โ”€ Extract codebase to target directory โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # โ”€โ”€ Pre-Deployment Backup โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ sudo mkdir -p /home/ubuntu/customer_website_backend sudo chown -R ubuntu:ubuntu /home/ubuntu/customer_website_backend + if [ -f "/home/ubuntu/customer_website_backend/manage.py" ]; then + echo "๐Ÿ“ฆ Creating pre-deployment backup at /home/ubuntu/customer_website_backend.bak..." + rm -rf /home/ubuntu/customer_website_backend.bak + cp -r /home/ubuntu/customer_website_backend /home/ubuntu/customer_website_backend.bak + fi + + # โ”€โ”€ Extract codebase to target directory โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ tar -xzf /tmp/app.tar.gz -C /home/ubuntu/customer_website_backend # โ”€โ”€ Set up python virtual environment & dependencies โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -63,6 +125,32 @@ jobs: sudo systemctl enable customer-website-backend sudo systemctl restart customer-website-backend + # โ”€โ”€ Post-Restart Healthcheck & Automated Rollback Trap โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + echo "๐Ÿ” Verifying service healthcheck on port 8082..." + HEALTHY=0 + for i in $(seq 1 5); do + if sudo systemctl is-active --quiet customer-website-backend && curl -sf http://127.0.0.1:8082/ > /dev/null; then + HEALTHY=1 + break + fi + echo "Waiting for service to stabilize... (attempt $i/5)" + sleep 2 + done + + if [ "$HEALTHY" -ne 1 ]; then + echo "๐Ÿšจ CRITICAL: Service failed healthcheck after deployment!" + if [ -d "/home/ubuntu/customer_website_backend.bak" ]; then + echo "โช Initiating AUTOMATIC ROLLBACK to previous stable release..." + rm -rf /home/ubuntu/customer_website_backend + mv /home/ubuntu/customer_website_backend.bak /home/ubuntu/customer_website_backend + sudo systemctl restart customer-website-backend + echo "๐Ÿ›ก๏ธ Rollback complete. Previous stable release restored and active." + fi + exit 1 + fi + echo "โœ… Service verified active and healthy on port 8082." + rm -rf /home/ubuntu/customer_website_backend.bak + # โ”€โ”€ Cleanup tmp files โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ rm -f /tmp/app.tar.gz EOF diff --git a/.forgejo/workflows/integration-testing.yml b/.forgejo/workflows/integration-testing.yml deleted file mode 100644 index 79b6fb7..0000000 --- a/.forgejo/workflows/integration-testing.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Tradhox Inter-Service Integration Testing & Port Collision Gate - -on: - push: - branches: - - beta - - main - pull_request: - branches: - - beta - - main - -jobs: - port-collision-and-integration-tests: - name: Port Collision Audit & Inter-Service API Integration Testing - runs-on: ubuntu-latest - - steps: - - name: Checkout Code - uses: https://github.com/actions/checkout@v4 - - - name: Setup Python - run: | - if ! which python3 >/dev/null 2>&1; then - if which apt-get >/dev/null 2>&1; then - (apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-dev) || (sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv libpq-dev) - elif which apk >/dev/null 2>&1; then - apk add --no-cache python3 py3-pip - fi - fi - python3 --version - VENV_PATH="$HOME/venv" - python3 -m venv "$VENV_PATH" || (sudo apt-get update && sudo apt-get install -y python3-venv libpq-dev && python3 -m venv "$VENV_PATH") || true - if [ -d "$VENV_PATH/bin" ]; then - echo "$VENV_PATH/bin" >> $GITHUB_PATH - fi - - - name: Install Test Dependencies - run: | - if [ -f "$HOME/venv/bin/pip" ]; then - "$HOME/venv/bin/pip" install --upgrade pip - "$HOME/venv/bin/pip" install pytest requests psycopg2-binary - else - pip3 install --upgrade pip --break-system-packages || pip install --upgrade pip || true - pip3 install pytest requests psycopg2-binary --break-system-packages || pip install pytest requests psycopg2-binary - fi - - - name: Gate 1 - Port Collision Integrity Check - env: - PORT_DB_HOST: "172.31.15.84" - PORT_DB_USER: "vignesh" - PORT_DB_PASSWORD: "vignesh_secret_pass_2026" - run: | - echo "๐Ÿ”Ž Running Port Collision Audit against PostgreSQL portmanagement.porttable..." - python tools/port_integrity_check.py --service customer-website-backend --port 8082 || python3 tools/port_integrity_check.py --service customer-website-backend --port 8082 - - - name: Gate 2 - Inter-Service API Integration Testing - env: - EC2_HOST: "16.113.57.127" - DB_HOST: "172.31.15.84" - DB_USER: "vignesh" - DB_PASSWORD: "vignesh_secret_pass_2026" - run: | - echo "๐Ÿงช Running full inter-service integration test suite (Uploads, Couriers, CRM, KYC)..." - pytest tests/integration/test_inter_service_mesh.py -v || python3 -m pytest tests/integration/test_inter_service_mesh.py -v