ci(deploy): enforce integration-tests dependency gate and auto-rollback in beta workflow

2026-11-12 14:09:33 +05:30 · 2026-11-12 14:09:33 +05:30 · c655be9c65
commit c655be9c65
parent 89aa750e67
2 changed files with 98 additions and 75 deletions
--- a/.forgejo/workflows/deploy-beta.yaml
+++ b/.forgejo/workflows/deploy-beta.yaml
@ -6,7 +6,61 @@ on:
      - beta

 jobs:
+  integration-tests:
+    name: Port Collision Audit & Inter-Service API Integration Testing
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: https://github.com/actions/checkout@v4
+
+      - name: Setup Python
+        run: |
+          if ! which python3 >/dev/null 2>&1; then
+            if which apt-get >/dev/null 2>&1; then
+              (apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-dev) || (sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv libpq-dev)
+            elif which apk >/dev/null 2>&1; then
+              apk add --no-cache python3 py3-pip
+            fi
+          fi
+          python3 --version
+          VENV_PATH="$HOME/venv"
+          python3 -m venv "$VENV_PATH" || (sudo apt-get update && sudo apt-get install -y python3-venv libpq-dev && python3 -m venv "$VENV_PATH") || true
+          if [ -d "$VENV_PATH/bin" ]; then
+            echo "$VENV_PATH/bin" >> $GITHUB_PATH
+          fi
+
+      - name: Install Test Dependencies
+        run: |
+          if [ -f "$HOME/venv/bin/pip" ]; then
+            "$HOME/venv/bin/pip" install --upgrade pip
+            "$HOME/venv/bin/pip" install pytest requests psycopg2-binary
+          else
+            pip3 install --upgrade pip --break-system-packages || pip install --upgrade pip || true
+            pip3 install pytest requests psycopg2-binary --break-system-packages || pip install pytest requests psycopg2-binary
+          fi
+
+      - name: Gate 1 - Port Collision Integrity Check
+        env:
+          PORT_DB_HOST: "172.31.15.84"
+          PORT_DB_USER: "vignesh"
+          PORT_DB_PASSWORD: "vignesh_secret_pass_2026"
+        run: |
+          echo "🔎 Running Port Collision Audit against PostgreSQL portmanagement.porttable..."
+          python tools/port_integrity_check.py --service crm-backend --port 8084 || python3 tools/port_integrity_check.py --service crm-backend --port 8084
+
+      - name: Gate 2 - Inter-Service API Integration Testing
+        env:
+          EC2_HOST: "16.113.57.127"
+          DB_HOST: "172.31.15.84"
+          DB_USER: "vignesh"
+          DB_PASSWORD: "vignesh_secret_pass_2026"
+        run: |
+          echo "🧪 Running full inter-service integration test suite (Uploads, Couriers, CRM, KYC)..."
+          pytest tests/integration/test_inter_service_mesh.py -v || python3 -m pytest tests/integration/test_inter_service_mesh.py -v
+
  deploy:
+    name: Deploy to Beta EC2
+    needs: [integration-tests]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
@ -53,7 +107,8 @@ jobs:

      - name: Deploy to Beta EC2
        run: |
-          apk add --no-cache openssh-client aws-cli
+          which apk >/dev/null 2>&1 && apk add --no-cache openssh-client aws-cli || true
+          which apt-get >/dev/null 2>&1 && (apt-get update && apt-get install -y openssh-client) || true
          mkdir -p ~/.ssh
          echo "${{ secrets.BETA_EC2_SSH_KEY }}" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
@ -70,19 +125,26 @@ jobs:
            sudo apt-get install -y python3-pip python3-venv python3-dev default-libmysqlclient-dev pkg-config libpq-dev build-essential libcairo2 libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 shared-mime-info
            sudo corepack enable

-            # ── 3. Extract backend codebase ───────────────────────────────
+            # ── 3. Pre-Deployment Backup ─────────────────────────────────
            sudo mkdir -p /home/ubuntu/crmbackend
            sudo chown -R ubuntu:ubuntu /home/ubuntu/crmbackend
+            if [ -f "/home/ubuntu/crmbackend/manage.py" ]; then
+              echo "📦 Creating pre-deployment backup at /home/ubuntu/crmbackend.bak..."
+              rm -rf /home/ubuntu/crmbackend.bak
+              cp -r /home/ubuntu/crmbackend /home/ubuntu/crmbackend.bak
+            fi
+
+            # ── 4. Extract backend codebase ───────────────────────────────
            tar -xzf /tmp/app.tar.gz -C /home/ubuntu/crmbackend

-            # ── 4. Set up python virtual environment & dependencies ────────
+            # ── 5. Set up python virtual environment & dependencies ────────
            cd /home/ubuntu/crmbackend
            mkdir -p logs
            python3 -m venv venv
            ./venv/bin/pip install --upgrade pip
            ./venv/bin/pip install -r requirements.txt

-            # ── 5. Configure environment variables ─────────────────────────
+            # ── 6. Configure environment variables ─────────────────────────
            echo "DEBUG=0" > .env
            echo "CRM_DB_NAME=crmprofile_beta" >> .env
            echo "CRM_DB_USER=vignesh" >> .env
@ -105,16 +167,42 @@ jobs:
            echo "PRODUCT_DB_HOST=16.113.106.152" >> .env
            echo "PRODUCT_DB_PORT=5432" >> .env

-            # ── 6. Run Database Migrations ─────────────────────────────────
+            # ── 7. Run Database Migrations ─────────────────────────────────
            ./venv/bin/python manage.py migrate --noinput

-            # ── 7. Configure and restart Backend Systemd service ───────────
+            # ── 8. Configure and restart Backend Systemd service ───────────
            sudo mv /tmp/crm-backend.service /etc/systemd/system/crm-backend.service
            sudo systemctl daemon-reload
            sudo systemctl enable crm-backend
            sudo systemctl restart crm-backend

-            # ── 8. Extract & Deploy Frontend ───────────────────────────────
+            # ── 9. Post-Restart Healthcheck & Automated Rollback Trap ───────
+            echo "🔍 Verifying backend healthcheck on port 8084..."
+            HEALTHY=0
+            for i in $(seq 1 5); do
+              if sudo systemctl is-active --quiet crm-backend && curl -sf http://127.0.0.1:8084/ > /dev/null; then
+                HEALTHY=1
+                break
+              fi
+              echo "Waiting for backend to stabilize... (attempt $i/5)"
+              sleep 2
+            done
+
+            if [ "$HEALTHY" -ne 1 ]; then
+              echo "🚨 CRITICAL: CRM Backend failed healthcheck after deployment!"
+              if [ -d "/home/ubuntu/crmbackend.bak" ]; then
+                echo "⏪ Initiating AUTOMATIC ROLLBACK to previous stable release..."
+                rm -rf /home/ubuntu/crmbackend
+                mv /home/ubuntu/crmbackend.bak /home/ubuntu/crmbackend
+                sudo systemctl restart crm-backend
+                echo "🛡️ Rollback complete. Previous stable release restored and active."
+              fi
+              exit 1
+            fi
+            echo "✅ CRM Backend verified active and healthy on port 8084."
+            rm -rf /home/ubuntu/crmbackend.bak
+
+            # ── 10. Extract & Deploy Frontend ───────────────────────────────
            sudo mkdir -p /home/ubuntu/crmfrontend
            sudo chown -R ubuntu:ubuntu /home/ubuntu/crmfrontend
            tar -xzf /tmp/frontend.tar.gz -C /home/ubuntu/crmfrontend/
@ -127,16 +215,16 @@ jobs:
            sudo systemctl enable crm-frontend
            sudo systemctl restart crm-frontend

-            # ── 9. Configure CloudWatch Agent ──────────────────────────────
+            # ── 11. Configure CloudWatch Agent ──────────────────────────────
            sudo mv /tmp/amazon-cloudwatch-agent.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
            sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
              -a fetch-config -m ec2 -s \
              -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json

-            # ── 10. Configure Nginx Proxy & Reload ─────────────────────────
+            # ── 12. Configure Nginx Proxy & Reload ─────────────────────────
            python3 /tmp/configure_nginx_proxy.py
            sudo nginx -t && sudo systemctl reload nginx

-            # ── 11. Cleanup tmp files ──────────────────────────────────────
+            # ── 13. Cleanup tmp files ──────────────────────────────────────
            rm -f /tmp/app.tar.gz /tmp/frontend.tar.gz /tmp/configure_nginx_proxy.py
          EOF
--- a/.forgejo/workflows/integration-testing.yml
+++ b/.forgejo/workflows/integration-testing.yml
@ -1,65 +0,0 @@
-name: Tradhox Inter-Service Integration Testing & Port Collision Gate
-
-on:
-  push:
-    branches:
-      - beta
-      - main
-  pull_request:
-    branches:
-      - beta
-      - main
-
-jobs:
-  port-collision-and-integration-tests:
-    name: Port Collision Audit & Inter-Service API Integration Testing
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout Code
-        uses: https://github.com/actions/checkout@v4
-
-      - name: Setup Python
-        run: |
-          if ! which python3 >/dev/null 2>&1; then
-            if which apt-get >/dev/null 2>&1; then
-              (apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-dev) || (sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv libpq-dev)
-            elif which apk >/dev/null 2>&1; then
-              apk add --no-cache python3 py3-pip
-            fi
-          fi
-          python3 --version
-          VENV_PATH="$HOME/venv"
-          python3 -m venv "$VENV_PATH" || (sudo apt-get update && sudo apt-get install -y python3-venv libpq-dev && python3 -m venv "$VENV_PATH") || true
-          if [ -d "$VENV_PATH/bin" ]; then
-            echo "$VENV_PATH/bin" >> $GITHUB_PATH
-          fi
-
-      - name: Install Test Dependencies
-        run: |
-          if [ -f "$HOME/venv/bin/pip" ]; then
-            "$HOME/venv/bin/pip" install --upgrade pip
-            "$HOME/venv/bin/pip" install pytest requests psycopg2-binary
-          else
-            pip3 install --upgrade pip --break-system-packages || pip install --upgrade pip || true
-            pip3 install pytest requests psycopg2-binary --break-system-packages || pip install pytest requests psycopg2-binary
-          fi
-
-      - name: Gate 1 - Port Collision Integrity Check
-        env:
-          PORT_DB_HOST: "172.31.15.84"
-          PORT_DB_USER: "vignesh"
-          PORT_DB_PASSWORD: "vignesh_secret_pass_2026"
-        run: |
-          echo "🔎 Running Port Collision Audit against PostgreSQL portmanagement.porttable..."
-          python tools/port_integrity_check.py --service crm-backend --port 8084 || python3 tools/port_integrity_check.py --service crm-backend --port 8084
-
-      - name: Gate 2 - Inter-Service API Integration Testing
-        env:
-          EC2_HOST: "16.113.57.127"
-          DB_HOST: "172.31.15.84"
-          DB_USER: "vignesh"
-          DB_PASSWORD: "vignesh_secret_pass_2026"
-        run: |
-          echo "🧪 Running full inter-service integration test suite (Uploads, Couriers, CRM, KYC)..."
-          pytest tests/integration/test_inter_service_mesh.py -v || python3 -m pytest tests/integration/test_inter_service_mesh.py -v