feat: initialize Django searchengine microservice with Typesense integration and CI/CD workflow

.forgejo/workflows/beta.yml

@@ -0,0 +1,145 @@
+name: Deploy Beta (NATIVE)
+
+on:
+  push:
+    branches:
+      - beta
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    container:
+      image: node:22-alpine
+    steps:
+      - name: Install git
+        run: apk add --no-cache git
+
+      - name: Checkout Code
+        uses: https://github.com/actions/checkout@v4
+
+      - name: Generate CloudWatch and Nginx Configs Locally
+        run: |
+          mkdir -p tmp
+          cat << 'EOF' > tmp/amazon-cloudwatch-agent.json
+          {
+            "agent": { "metrics_collection_interval": 60, "run_as_user": "root" },
+            "metrics": {
+              "metrics_collected": {
+                "disk": { "measurement": ["used_percent"], "metrics_collection_interval": 60, "resources": ["/"] },
+                "mem":  { "measurement": ["mem_used_percent"], "metrics_collection_interval": 60 }
+              }
+            },
+            "logs": {
+              "logs_collected": {
+                "files": {
+                  "collect_list": [
+                    { "file_path": "/home/ubuntu/searchengine/logs/app.log",      "log_group_name": "/searchengine/app",      "log_stream_name": "{instance_id}", "retention_in_days": 30 },
+                    { "file_path": "/home/ubuntu/searchengine/logs/error.log",    "log_group_name": "/searchengine/error",    "log_stream_name": "{instance_id}", "retention_in_days": 30 },
+                    { "file_path": "/home/ubuntu/searchengine/logs/requests.log", "log_group_name": "/searchengine/requests", "log_stream_name": "{instance_id}", "retention_in_days": 30 },
+                    { "file_path": "/home/ubuntu/searchengine/logs/gunicorn-access.log", "log_group_name": "/searchengine/gunicorn-access", "log_stream_name": "{instance_id}", "retention_in_days": 30 },
+                    { "file_path": "/home/ubuntu/searchengine/logs/gunicorn-error.log",  "log_group_name": "/searchengine/gunicorn-error",  "log_stream_name": "{instance_id}", "retention_in_days": 30 }
+                  ]
+                }
+              }
+            }
+          }
+          EOF
+          cat << 'EOF' > tmp/nginx-proxy.conf
+          server {
+              server_name betasearch-api.tradhox.com search-api.tradhox.com;
+              
+              location / {
+                  proxy_pass http://127.0.0.1:8087;
+                  proxy_set_header Host $host;
+                  proxy_set_header X-Real-IP $remote_addr;
+                  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                  proxy_set_header X-Forwarded-Proto $scheme;
+              }
+
+              listen 443 ssl;
+              ssl_certificate /etc/letsencrypt/live/betapartner-api.tradhox.com/fullchain.pem;
+              ssl_certificate_key /etc/letsencrypt/live/betapartner-api.tradhox.com/privkey.pem;
+              include /etc/letsencrypt/options-ssl-nginx.conf;
+              ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+          }
+
+          server {
+              listen 80;
+              server_name betasearch-api.tradhox.com search-api.tradhox.com;
+              return 301 https://$host$request_uri;
+          }
+          EOF
+
+      - name: Package Application
+        run: |
+          tar -czf app.tar.gz --exclude=.git --exclude=tmp .
+
+      - name: Deploy to Beta EC2
+        run: |
+          apk add --no-cache openssh-client aws-cli
+          mkdir -p ~/.ssh
+          echo "${{ secrets.BETA_EC2_SSH_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+
+          # Copy app archive and configuration files
+          scp -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa app.tar.gz tmp/amazon-cloudwatch-agent.json tmp/nginx-proxy.conf searchengine.service ${{ secrets.BETA_EC2_USERNAME }}@${{ secrets.BETA_EC2_HOST }}:/tmp/
+
+          # Execute deployment commands on remote server
+          ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa ${{ secrets.BETA_EC2_USERNAME }}@${{ secrets.BETA_EC2_HOST }} << 'EOF'
+            set -e
+
+            # ── 1. Install Host Dependencies ──────────────────────────────
+            sudo apt-get update
+            sudo apt-get install -y python3-pip python3-venv python3-dev pkg-config libpq-dev build-essential nginx
+
+            # ── 2. Install Typesense Server Binary if not present ──────────
+            if ! command -v typesense-server &> /dev/null; then
+              echo "Installing Typesense Server v27.1..."
+              curl -O https://dl.typesense.org/releases/27.1/typesense-server-27.1-amd64.deb
+              sudo dpkg -i typesense-server-27.1-amd64.deb
+              rm typesense-server-27.1-amd64.deb
+            fi
+
+            # Ensure Typesense directory & config exists
+            sudo mkdir -p /var/lib/typesense
+            cat << 'TYPESENSE_CONF' | sudo tee /etc/typesense/typesense-server.ini
+            api-key = vignesh_typesense_master_key_2026
+            data-dir = /var/lib/typesense
+            log-dir = /var/log/typesense
+            api-port = 8108
+            listen-address = 127.0.0.1
+            TYPESENSE_CONF
+
+            sudo systemctl enable typesense-server
+            sudo systemctl restart typesense-server || true
+
+            # ── 3. Extract codebase to target directory ───────────────────
+            sudo mkdir -p /home/ubuntu/searchengine
+            sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine
+            tar -xzf /tmp/app.tar.gz -C /home/ubuntu/searchengine
+
+            # ── 4. Set up python virtual environment & dependencies ────────
+            cd /home/ubuntu/searchengine
+            python3 -m venv venv
+            source venv/bin/activate
+            pip install --upgrade pip
+            pip install -r requirements.txt
+
+            # ── 5. Systemd Service Deployment ──────────────────────────────
+            sudo cp /tmp/searchengine.service /etc/systemd/system/searchengine.service
+            sudo systemctl daemon-reload
+            sudo systemctl enable searchengine.service
+            sudo systemctl restart searchengine.service
+
+            # ── 6. Nginx Config ───────────────────────────────────────────
+            sudo cp /tmp/nginx-proxy.conf /etc/nginx/sites-available/searchengine
+            sudo ln -sf /etc/nginx/sites-available/searchengine /etc/nginx/sites-enabled/searchengine
+            sudo nginx -t && sudo systemctl reload nginx
+
+            # ── 7. CloudWatch Agent Config ────────────────────────────────
+            sudo mkdir -p /opt/aws/amazon-cloudwatch-agent/etc/
+            sudo cp /tmp/amazon-cloudwatch-agent.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
+            sudo systemctl restart amazon-cloudwatch-agent || true
+
+            echo "searchengine microservice successfully deployed!"
+          EOF

api/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+class ApiConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'api'

api/management/commands/reindex_typesense.py

@@ -0,0 +1,29 @@
+from django.core.management.base import BaseCommand
+from django.db import connection
+from api.typesense_client import TypesenseService
+import time
+
+class Command(BaseCommand):
+    help = "Reindex all products from PostgreSQL database into Typesense search index."
+
+    def handle(self, *args, **options):
+        self.stdout.write(self.style.SUCCESS("Starting Typesense reindex..."))
+        TypesenseService.ensure_schema()
+        count = 0
+
+        with connection.cursor() as cursor:
+            cursor.execute("""
+                SELECT id, sku, title, description, category, supplier_id, price, 
+                       compare_at_price, stock, image, is_gi_tagged, is_active, status
+                FROM api_product
+            """)
+            columns = [col[0] for col in cursor.description]
+            rows = cursor.fetchall()
+            for row in rows:
+                item = dict(zip(columns, row))
+                item['created_at'] = int(time.time())
+                TypesenseService.index_product(item)
+                count += 1
+                self.stdout.write(f"Indexed product SKU: {item.get('sku')} | Title: {item.get('title')}")
+
+        self.stdout.write(self.style.SUCCESS(f"Successfully reindexed {count} products into Typesense."))

api/models.py

@@ -0,0 +1,11 @@
+from django.db import models
+
+# SearchEngine microservice references products directly from PostgreSQL productprofile db or typesense index
+class ProductIndexLog(models.Model):
+    sku = models.CharField(max_length=100, unique=True)
+    status = models.CharField(max_length=50, default='synced')
+    last_synced_at = models.DateTimeField(auto_now=True)
+    error = models.TextField(blank=True, null=True)
+
+    def __str__(self):
+        return f"{self.sku} - {self.status}"

api/typesense_client.py

@@ -0,0 +1,109 @@
+import typesense
+from django.conf import settings
+import logging
+
+logger = logging.getLogger(__name__)
+
+PRODUCTS_SCHEMA = {
+    'name': 'products',
+    'fields': [
+        {'name': 'id', 'type': 'string'},
+        {'name': 'title', 'type': 'string'},
+        {'name': 'description', 'type': 'string', 'optional': True},
+        {'name': 'category', 'type': 'string', 'facet': True},
+        {'name': 'supplier_id', 'type': 'int32', 'facet': True},
+        {'name': 'supplier_name', 'type': 'string', 'optional': True},
+        {'name': 'price', 'type': 'float', 'facet': True},
+        {'name': 'compare_at_price', 'type': 'float', 'optional': True},
+        {'name': 'stock', 'type': 'int32'},
+        {'name': 'sku', 'type': 'string'},
+        {'name': 'image', 'type': 'string', 'optional': True},
+        {'name': 'is_gi_tagged', 'type': 'bool', 'facet': True},
+        {'name': 'is_active', 'type': 'bool', 'facet': True},
+        {'name': 'status', 'type': 'string', 'facet': True},
+        {'name': 'created_at', 'type': 'int64', 'optional': True},
+    ],
+    'default_sorting_field': 'created_at'
+}
+
+class TypesenseService:
+    _client = None
+
+    @classmethod
+    def get_client(cls):
+        if cls._client is None:
+            cfg = getattr(settings, 'TYPESENSE_CONFIG', {})
+            cls._client = typesense.Client({
+                'nodes': [{
+                    'host': cfg.get('HOST', '127.0.0.1'),
+                    'port': str(cfg.get('PORT', '8108')),
+                    'protocol': cfg.get('PROTOCOL', 'http')
+                }],
+                'api_key': cfg.get('API_KEY', 'vignesh_typesense_master_key_2026'),
+                'connection_timeout_seconds': cfg.get('TIMEOUT', 5.0)
+            })
+        return cls._client
+
+    @classmethod
+    def ensure_schema(cls):
+        client = cls.get_client()
+        try:
+            client.collections['products'].retrieve()
+            logger.info("Typesense 'products' collection already exists.")
+        except Exception:
+            logger.info("Creating Typesense 'products' collection schema...")
+            client.collections.create(PRODUCTS_SCHEMA)
+
+    @classmethod
+    def index_product(cls, product_dict):
+        cls.ensure_schema()
+        client = cls.get_client()
+        doc = {
+            'id': str(product_dict.get('id', product_dict.get('sku'))),
+            'title': str(product_dict.get('title', '')),
+            'description': str(product_dict.get('description', '') or ''),
+            'category': str(product_dict.get('category', 'Uncategorized')),
+            'supplier_id': int(product_dict.get('supplier_id', 0)),
+            'supplier_name': str(product_dict.get('supplier_name', '') or ''),
+            'price': float(product_dict.get('price', 0.0)),
+            'compare_at_price': float(product_dict.get('compare_at_price', 0.0) or 0.0),
+            'stock': int(product_dict.get('stock', 0)),
+            'sku': str(product_dict.get('sku', '')),
+            'image': str(product_dict.get('image', '') or ''),
+            'is_gi_tagged': bool(product_dict.get('is_gi_tagged', False)),
+            'is_active': bool(product_dict.get('is_active', True)),
+            'status': str(product_dict.get('status', 'active')),
+            'created_at': int(product_dict.get('created_at', 0))
+        }
+        return client.collections['products'].documents.upsert(doc)
+
+    @classmethod
+    def delete_product(cls, product_id):
+        client = cls.get_client()
+        try:
+            return client.collections['products'].documents[str(product_id)].delete()
+        except Exception as e:
+            logger.warning(f"Could not delete document {product_id} from Typesense: {e}")
+            return None
+
+    @classmethod
+    def search_products(cls, query="*", filter_by="", sort_by="", page=1, per_page=20):
+        cls.ensure_schema()
+        client = cls.get_client()
+        search_parameters = {
+            'q': query if query else '*',
+            'query_by': 'title,description,category,sku',
+            'facet_by': 'category,is_gi_tagged,supplier_id',
+            'page': page,
+            'per_page': per_page,
+            'num_typos': 2,
+            'prefix': True
+        }
+        if filter_by:
+            search_parameters['filter_by'] = filter_by
+        if sort_by:
+            search_parameters['sort_by'] = sort_by
+        else:
+            search_parameters['sort_by'] = 'created_at:desc'
+
+        return client.collections['products'].documents.search(search_parameters)

api/urls.py

@@ -0,0 +1,10 @@
+from django.urls import path
+from .views import ProductSearchView, IndexProductView, DeleteProductView, ReindexCatalogView
+
+urlpatterns = [
+    path('search/', ProductSearchView.as_view(), name='product-search'),
+    path('products/search/', ProductSearchView.as_view(), name='product-search-alias'),
+    path('index/product/', IndexProductView.as_view(), name='index-product'),
+    path('index/product/<str:product_id>/', DeleteProductView.as_view(), name='delete-product'),
+    path('index/reindex/', ReindexCatalogView.as_view(), name='reindex-catalog'),
+]

api/views.py

@@ -0,0 +1,123 @@
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from rest_framework import status
+from django.db import connection
+from .typesense_client import TypesenseService
+import logging
+import time
+
+logger = logging.getLogger(__name__)
+
+class ProductSearchView(APIView):
+    def get(self, request):
+        try:
+            q = request.query_params.get('q', '*').strip()
+            category = request.query_params.get('category', '').strip()
+            min_price = request.query_params.get('min_price')
+            max_price = request.query_params.get('max_price')
+            is_gi_tagged = request.query_params.get('is_gi_tagged')
+            supplier_id = request.query_params.get('supplier_id')
+            sort_by_param = request.query_params.get('sort_by', 'newest')
+            page = int(request.query_params.get('page', 1))
+            per_page = int(request.query_params.get('per_page', 20))
+
+            filter_clauses = ['is_active:=true']
+
+            if category:
+                filter_clauses.append(f'category:="{category}"')
+            if supplier_id:
+                filter_clauses.append(f'supplier_id:={supplier_id}')
+            if is_gi_tagged is not None and is_gi_tagged.lower() in ('true', '1'):
+                filter_clauses.append('is_gi_tagged:=true')
+            
+            if min_price is not None or max_price is not None:
+                p_min = float(min_price) if min_price else 0.0
+                p_max = float(max_price) if max_price else 9999999.0
+                filter_clauses.append(f'price:[{p_min}..{p_max}]')
+
+            filter_by_str = ' && '.join(filter_clauses)
+
+            sort_by_str = 'created_at:desc'
+            if sort_by_param == 'price_asc':
+                sort_by_str = 'price:asc'
+            elif sort_by_param == 'price_desc':
+                sort_by_str = 'price:desc'
+            elif sort_by_param == 'relevance' and q != '*':
+                sort_by_str = ''
+
+            res = TypesenseService.search_products(
+                query=q,
+                filter_by=filter_by_str,
+                sort_by=sort_by_str,
+                page=page,
+                per_page=per_page
+            )
+
+            hits = res.get('hits', [])
+            found = res.get('found', 0)
+            total_pages = (found + per_page - 1) // per_page if per_page > 0 else 1
+
+            results = [hit.get('document', {}) for hit in hits]
+            facet_counts = res.get('facet_counts', [])
+
+            return Response({
+                'count': found,
+                'page': page,
+                'per_page': per_page,
+                'total_pages': total_pages,
+                'facets': facet_counts,
+                'results': results
+            }, status=status.HTTP_200_OK)
+        except Exception as e:
+            logger.error(f"Typesense Search Error: {e}", exc_info=True)
+            return Response({'error': str(e), 'results': [], 'count': 0}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+class IndexProductView(APIView):
+    def post(self, request):
+        try:
+            product_data = request.data
+            if not product_data or ('id' not in product_data and 'sku' not in product_data):
+                return Response({'error': 'Product id or sku is required'}, status=status.HTTP_400_BAD_REQUEST)
+            
+            res = TypesenseService.index_product(product_data)
+            return Response({'status': 'indexed', 'result': res}, status=status.HTTP_200_OK)
+        except Exception as e:
+            logger.error(f"Product Index Error: {e}", exc_info=True)
+            return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+class DeleteProductView(APIView):
+    def delete(self, request, product_id):
+        try:
+            res = TypesenseService.delete_product(product_id)
+            return Response({'status': 'deleted', 'result': res}, status=status.HTTP_200_OK)
+        except Exception as e:
+            logger.error(f"Product Delete Error: {e}", exc_info=True)
+            return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+class ReindexCatalogView(APIView):
+    def post(self, request):
+        try:
+            indexed_count = 0
+            TypesenseService.ensure_schema()
+            
+            with connection.cursor() as cursor:
+                cursor.execute("""
+                    SELECT id, sku, title, description, category, supplier_id, price, 
+                           compare_at_price, stock, image, is_gi_tagged, is_active, status
+                    FROM api_product
+                """)
+                columns = [col[0] for col in cursor.description]
+                rows = cursor.fetchall()
+                for row in rows:
+                    item = dict(zip(columns, row))
+                    item['created_at'] = int(time.time())
+                    TypesenseService.index_product(item)
+                    indexed_count += 1
+
+            return Response({'status': 'completed', 'total_indexed': indexed_count}, status=status.HTTP_200_OK)
+        except Exception as e:
+            logger.error(f"Full Reindex Error: {e}", exc_info=True)
+            return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

config/asgi.py

@@ -0,0 +1,4 @@
+import os
+from django.core.asgi import get_asgi_application
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+application = get_asgi_application()

config/settings.py

@@ -0,0 +1,90 @@
+# Django settings for searchengine microservice
+import os
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+SECRET_KEY = os.environ.get('SECRET_KEY', 'django-insecure-tradhox-searchengine-key-2026')
+DEBUG = os.environ.get('DEBUG', 'True').lower() in ('true', '1')
+ALLOWED_HOSTS = ['*']
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'corsheaders',
+    'rest_framework',
+    'api',
+]
+
+MIDDLEWARE = [
+    'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'config.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'config.wsgi.application'
+
+# Database Configuration (PostgreSQL Exclusive)
+db_host = os.environ.get('DB_HOST', '16.113.106.152')
+db_port = int(os.environ.get('DB_PORT', '5432'))
+db_user = os.environ.get('DB_USER', 'vignesh')
+db_pass = os.environ.get('DB_PASSWORD', 'vignesh_secret_pass_2026')
+env_name = os.environ.get('ENVIRONMENT', 'beta')
+db_suffix = f'_{env_name}'
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': f'productprofile{db_suffix}',
+        'USER': db_user,
+        'PASSWORD': db_pass,
+        'HOST': db_host,
+        'PORT': str(db_port),
+    }
+}
+
+AUTH_PASSWORD_VALIDATORS = []
+
+LANGUAGE_CODE = 'en-us'
+TIME_ZONE = 'UTC'
+USE_I18N = True
+USE_TZ = True
+
+STATIC_URL = 'static/'
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+CORS_ALLOW_ALL_ORIGINS = True
+
+TYPESENSE_CONFIG = {
+    'HOST': os.environ.get('TYPESENSE_HOST', '127.0.0.1'),
+    'PORT': os.environ.get('TYPESENSE_PORT', '8108'),
+    'PROTOCOL': os.environ.get('TYPESENSE_PROTOCOL', 'http'),
+    'API_KEY': os.environ.get('TYPESENSE_API_KEY', 'vignesh_typesense_master_key_2026'),
+    'TIMEOUT': 5.0
+}

config/urls.py

@@ -0,0 +1,7 @@
+from django.contrib import admin
+from django.urls import path, include
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+    path('api/', include('api.urls')),
+]

config/wsgi.py

@@ -0,0 +1,4 @@
+import os
+from django.core.wsgi import get_wsgi_application
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+application = get_wsgi_application()

manage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+import os
+import sys
+
+def main():
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -0,0 +1,7 @@
+Django==6.1
+djangorestframework
+django-cors-headers
+gunicorn==26.0.0
+psycopg2-binary
+requests
+typesense>=0.21.0

searchengine.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Tradhox Search Engine Microservice Daemon
+After=network.target typesense-server.service
+
+[Service]
+User=ubuntu
+Group=ubuntu
+WorkingDirectory=/home/ubuntu/searchengine
+ExecStart=/home/ubuntu/searchengine/venv/bin/gunicorn --workers 3 --bind 0.0.0.0:8087 config.wsgi:application
+Restart=always
+RestartSec=5
+StandardOutput=append:/home/ubuntu/searchengine/logs/gunicorn-access.log
+StandardError=append:/home/ubuntu/searchengine/logs/gunicorn-error.log
+
+[Install]
+WantedBy=multi-user.target