ci(deploy): enforce integration-tests dependency gate and auto-rollback in beta workflow
This commit is contained in:
parent
ef6130bcea
commit
43f71bbb55
2 changed files with 90 additions and 67 deletions
|
|
@ -6,7 +6,61 @@ on:
|
|||
- beta
|
||||
|
||||
jobs:
|
||||
integration-tests:
|
||||
name: Port Collision Audit & Inter-Service API Integration Testing
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
run: |
|
||||
if ! which python3 >/dev/null 2>&1; then
|
||||
if which apt-get >/dev/null 2>&1; then
|
||||
(apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-dev) || (sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv libpq-dev)
|
||||
elif which apk >/dev/null 2>&1; then
|
||||
apk add --no-cache python3 py3-pip
|
||||
fi
|
||||
fi
|
||||
python3 --version
|
||||
VENV_PATH="$HOME/venv"
|
||||
python3 -m venv "$VENV_PATH" || (sudo apt-get update && sudo apt-get install -y python3-venv libpq-dev && python3 -m venv "$VENV_PATH") || true
|
||||
if [ -d "$VENV_PATH/bin" ]; then
|
||||
echo "$VENV_PATH/bin" >> $GITHUB_PATH
|
||||
fi
|
||||
|
||||
- name: Install Test Dependencies
|
||||
run: |
|
||||
if [ -f "$HOME/venv/bin/pip" ]; then
|
||||
"$HOME/venv/bin/pip" install --upgrade pip
|
||||
"$HOME/venv/bin/pip" install pytest requests psycopg2-binary
|
||||
else
|
||||
pip3 install --upgrade pip --break-system-packages || pip install --upgrade pip || true
|
||||
pip3 install pytest requests psycopg2-binary --break-system-packages || pip install pytest requests psycopg2-binary
|
||||
fi
|
||||
|
||||
- name: Gate 1 - Port Collision Integrity Check
|
||||
env:
|
||||
PORT_DB_HOST: "172.31.15.84"
|
||||
PORT_DB_USER: "vignesh"
|
||||
PORT_DB_PASSWORD: "vignesh_secret_pass_2026"
|
||||
run: |
|
||||
echo "🔎 Running Port Collision Audit against PostgreSQL portmanagement.porttable..."
|
||||
python tools/port_integrity_check.py --service customer-website-backend --port 8082 || python3 tools/port_integrity_check.py --service customer-website-backend --port 8082
|
||||
|
||||
- name: Gate 2 - Inter-Service API Integration Testing
|
||||
env:
|
||||
EC2_HOST: "16.113.57.127"
|
||||
DB_HOST: "172.31.15.84"
|
||||
DB_USER: "vignesh"
|
||||
DB_PASSWORD: "vignesh_secret_pass_2026"
|
||||
run: |
|
||||
echo "🧪 Running full inter-service integration test suite (Uploads, Couriers, CRM, KYC)..."
|
||||
pytest tests/integration/test_inter_service_mesh.py -v || python3 -m pytest tests/integration/test_inter_service_mesh.py -v
|
||||
|
||||
deploy:
|
||||
name: Deploy to Beta EC2
|
||||
needs: [integration-tests]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
|
@ -18,7 +72,8 @@ jobs:
|
|||
|
||||
- name: Deploy to Beta EC2
|
||||
run: |
|
||||
apk add --no-cache openssh-client aws-cli
|
||||
which apk >/dev/null 2>&1 && apk add --no-cache openssh-client aws-cli || true
|
||||
which apt-get >/dev/null 2>&1 && (apt-get update && apt-get install -y openssh-client) || true
|
||||
mkdir -p ~/.ssh
|
||||
echo "${{ secrets.BETA_EC2_SSH_KEY }}" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
|
|
@ -34,9 +89,16 @@ jobs:
|
|||
sudo apt-get update
|
||||
sudo apt-get install -y python3-pip python3-venv python3-dev libpq-dev build-essential
|
||||
|
||||
# ── Extract codebase to target directory ───────────────────
|
||||
# ── Pre-Deployment Backup ─────────────────────────────────
|
||||
sudo mkdir -p /home/ubuntu/customer_website_backend
|
||||
sudo chown -R ubuntu:ubuntu /home/ubuntu/customer_website_backend
|
||||
if [ -f "/home/ubuntu/customer_website_backend/manage.py" ]; then
|
||||
echo "📦 Creating pre-deployment backup at /home/ubuntu/customer_website_backend.bak..."
|
||||
rm -rf /home/ubuntu/customer_website_backend.bak
|
||||
cp -r /home/ubuntu/customer_website_backend /home/ubuntu/customer_website_backend.bak
|
||||
fi
|
||||
|
||||
# ── Extract codebase to target directory ───────────────────
|
||||
tar -xzf /tmp/app.tar.gz -C /home/ubuntu/customer_website_backend
|
||||
|
||||
# ── Set up python virtual environment & dependencies ────────
|
||||
|
|
@ -63,6 +125,32 @@ jobs:
|
|||
sudo systemctl enable customer-website-backend
|
||||
sudo systemctl restart customer-website-backend
|
||||
|
||||
# ── Post-Restart Healthcheck & Automated Rollback Trap ───────
|
||||
echo "🔍 Verifying service healthcheck on port 8082..."
|
||||
HEALTHY=0
|
||||
for i in $(seq 1 5); do
|
||||
if sudo systemctl is-active --quiet customer-website-backend && curl -sf http://127.0.0.1:8082/ > /dev/null; then
|
||||
HEALTHY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for service to stabilize... (attempt $i/5)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [ "$HEALTHY" -ne 1 ]; then
|
||||
echo "🚨 CRITICAL: Service failed healthcheck after deployment!"
|
||||
if [ -d "/home/ubuntu/customer_website_backend.bak" ]; then
|
||||
echo "⏪ Initiating AUTOMATIC ROLLBACK to previous stable release..."
|
||||
rm -rf /home/ubuntu/customer_website_backend
|
||||
mv /home/ubuntu/customer_website_backend.bak /home/ubuntu/customer_website_backend
|
||||
sudo systemctl restart customer-website-backend
|
||||
echo "🛡️ Rollback complete. Previous stable release restored and active."
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Service verified active and healthy on port 8082."
|
||||
rm -rf /home/ubuntu/customer_website_backend.bak
|
||||
|
||||
# ── Cleanup tmp files ───────────────────────────────────────
|
||||
rm -f /tmp/app.tar.gz
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
name: Tradhox Inter-Service Integration Testing & Port Collision Gate
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- beta
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- beta
|
||||
- main
|
||||
|
||||
jobs:
|
||||
port-collision-and-integration-tests:
|
||||
name: Port Collision Audit & Inter-Service API Integration Testing
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
run: |
|
||||
if ! which python3 >/dev/null 2>&1; then
|
||||
if which apt-get >/dev/null 2>&1; then
|
||||
(apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-dev) || (sudo apt-get update && sudo apt-get install -y python3 python3-pip python3-venv libpq-dev)
|
||||
elif which apk >/dev/null 2>&1; then
|
||||
apk add --no-cache python3 py3-pip
|
||||
fi
|
||||
fi
|
||||
python3 --version
|
||||
VENV_PATH="$HOME/venv"
|
||||
python3 -m venv "$VENV_PATH" || (sudo apt-get update && sudo apt-get install -y python3-venv libpq-dev && python3 -m venv "$VENV_PATH") || true
|
||||
if [ -d "$VENV_PATH/bin" ]; then
|
||||
echo "$VENV_PATH/bin" >> $GITHUB_PATH
|
||||
fi
|
||||
|
||||
- name: Install Test Dependencies
|
||||
run: |
|
||||
if [ -f "$HOME/venv/bin/pip" ]; then
|
||||
"$HOME/venv/bin/pip" install --upgrade pip
|
||||
"$HOME/venv/bin/pip" install pytest requests psycopg2-binary
|
||||
else
|
||||
pip3 install --upgrade pip --break-system-packages || pip install --upgrade pip || true
|
||||
pip3 install pytest requests psycopg2-binary --break-system-packages || pip install pytest requests psycopg2-binary
|
||||
fi
|
||||
|
||||
- name: Gate 1 - Port Collision Integrity Check
|
||||
env:
|
||||
PORT_DB_HOST: "172.31.15.84"
|
||||
PORT_DB_USER: "vignesh"
|
||||
PORT_DB_PASSWORD: "vignesh_secret_pass_2026"
|
||||
run: |
|
||||
echo "🔎 Running Port Collision Audit against PostgreSQL portmanagement.porttable..."
|
||||
python tools/port_integrity_check.py --service customer-website-backend --port 8082 || python3 tools/port_integrity_check.py --service customer-website-backend --port 8082
|
||||
|
||||
- name: Gate 2 - Inter-Service API Integration Testing
|
||||
env:
|
||||
EC2_HOST: "16.113.57.127"
|
||||
DB_HOST: "172.31.15.84"
|
||||
DB_USER: "vignesh"
|
||||
DB_PASSWORD: "vignesh_secret_pass_2026"
|
||||
run: |
|
||||
echo "🧪 Running full inter-service integration test suite (Uploads, Couriers, CRM, KYC)..."
|
||||
pytest tests/integration/test_inter_service_mesh.py -v || python3 -m pytest tests/integration/test_inter_service_mesh.py -v
|
||||
Loading…
Reference in a new issue