feat: add search engine API with Typesense integration, Django models, and CI/CD deployment workflow

.forgejo/workflows/beta.yml

@@ -1,4 +1,4 @@
-name: Deploy Beta (NATIVE)
+name: Deploy Beta (NATIVE)
 
 on:
   push:
@@ -118,6 +118,8 @@ jobs:
             sudo mkdir -p /home/ubuntu/searchengine
             sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine
             tar -xzf /tmp/app.tar.gz -C /home/ubuntu/searchengine
+            mkdir -p /home/ubuntu/searchengine/logs
+            sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine/logs
 
             # ── 4. Set up python virtual environment & dependencies ────────
             cd /home/ubuntu/searchengine
@@ -125,14 +127,22 @@ jobs:
             source venv/bin/activate
             pip install --upgrade pip
             pip install -r requirements.txt
+            python manage.py migrate --noinput
 
             # ── 5. Systemd Service Deployment ──────────────────────────────
             sudo cp /tmp/searchengine.service /etc/systemd/system/searchengine.service
             sudo systemctl daemon-reload
             sudo systemctl enable searchengine.service
-            sudo systemctl restart searchengine.service
+            sudo systemctl restart searchengine.service || {
+              echo "searchengine.service failed to restart! Checking journalctl:"
+              sudo journalctl -u searchengine.service -n 50 --no-pager
+              exit 1
+            }
 
             # ── 6. Nginx Config ───────────────────────────────────────────
+            if [ -f "/etc/letsencrypt/live/betasearch-api.tradhox.com/fullchain.pem" ]; then
+              sed -i 's|/etc/letsencrypt/live/betapartner-api.tradhox.com|/etc/letsencrypt/live/betasearch-api.tradhox.com|g' /tmp/nginx-proxy.conf
+            fi
             sudo cp /tmp/nginx-proxy.conf /etc/nginx/sites-available/searchengine
             sudo ln -sf /etc/nginx/sites-available/searchengine /etc/nginx/sites-enabled/searchengine
             sudo nginx -t && sudo systemctl reload nginx

api/management/commands/reindex_typesense.py

@@ -1,4 +1,4 @@
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand
 from django.db import connection
 from api.typesense_client import TypesenseService
 import time
@@ -6,11 +6,24 @@ import time
 class Command(BaseCommand):
     help = "Reindex all products from PostgreSQL database into Typesense search index."
 
-    def handle(self, *args, **options):
-        self.stdout.write(self.style.SUCCESS("Starting Typesense reindex..."))
-        TypesenseService.ensure_schema()
-        count = 0
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--batch-size',
+            type=int,
+            default=250,
+            help='Number of records to index per batch request.'
+        )
 
+    def handle(self, *args, **options):
+        batch_size = options.get('batch_size', 250)
+        self.stdout.write(self.style.SUCCESS("Ensuring Typesense schema..."))
+        TypesenseService.ensure_schema(force=True)
+
+        count = 0
+        batch = []
+        now_ts = int(time.time())
+
+        self.stdout.write(self.style.SUCCESS("Fetching products from PostgreSQL (api_product)..."))
         with connection.cursor() as cursor:
             cursor.execute("""
                 SELECT id, sku, title, description, category, supplier_id, price, 
@@ -19,11 +32,23 @@ class Command(BaseCommand):
             """)
             columns = [col[0] for col in cursor.description]
             rows = cursor.fetchall()
-            for row in rows:
+            total_rows = len(rows)
+            self.stdout.write(f"Found {total_rows} products to index.")
+
+            for idx, row in enumerate(rows):
                 item = dict(zip(columns, row))
-                item['created_at'] = int(time.time())
-                TypesenseService.index_product(item)
-                count += 1
-                self.stdout.write(f"Indexed product SKU: {item.get('sku')} | Title: {item.get('title')}")
+                item['created_at'] = now_ts + idx
+                batch.append(item)
+
+                if len(batch) >= batch_size:
+                    TypesenseService.index_products_batch(batch)
+                    count += len(batch)
+                    self.stdout.write(f"Indexed batch: {count}/{total_rows} products...")
+                    batch = []
+
+            if batch:
+                TypesenseService.index_products_batch(batch)
+                count += len(batch)
+                self.stdout.write(f"Indexed final batch: {count}/{total_rows} products...")
 
         self.stdout.write(self.style.SUCCESS(f"Successfully reindexed {count} products into Typesense."))

api/migrations/0001_initial.py

@@ -0,0 +1,24 @@
+# Generated by Django 6.1 on 2026-10-15 14:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ProductIndexLog',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('sku', models.CharField(max_length=100, unique=True)),
+                ('status', models.CharField(default='synced', max_length=50)),
+                ('last_synced_at', models.DateTimeField(auto_now=True)),
+                ('error', models.TextField(blank=True, null=True)),
+            ],
+        ),
+    ]

api/tests.py

@@ -0,0 +1,134 @@
+from django.test import TestCase
+from rest_framework.test import APIClient
+from unittest.mock import patch, MagicMock
+from api.typesense_client import (
+    TypesenseService,
+    safe_str,
+    safe_int,
+    safe_float,
+    safe_bool,
+)
+
+class TypesenseServiceUnitTests(TestCase):
+    def test_safe_helpers(self):
+        self.assertEqual(safe_str(None, "default"), "default")
+        self.assertEqual(safe_str(123), "123")
+        self.assertEqual(safe_int(None, 10), 10)
+        self.assertEqual(safe_int("42"), 42)
+        self.assertEqual(safe_int("invalid", 5), 5)
+        self.assertEqual(safe_float(None, 3.14), 3.14)
+        self.assertEqual(safe_float("99.95"), 99.95)
+        self.assertEqual(safe_float("invalid", 0.0), 0.0)
+        self.assertTrue(safe_bool("true"))
+        self.assertTrue(safe_bool("1"))
+        self.assertFalse(safe_bool("false"))
+        self.assertFalse(safe_bool("0"))
+        self.assertFalse(safe_bool(None, False))
+
+    def test_prepare_product_doc_handles_nulls_gracefully(self):
+        raw_product = {
+            'id': 101,
+            'sku': 'TRDX-TEST-001',
+            'title': 'Test Silk Saree',
+            'description': None,
+            'category': None,
+            'supplier_id': None,
+            'supplier_name': None,
+            'price': None,
+            'compare_at_price': None,
+            'stock': None,
+            'image': None,
+            'is_gi_tagged': None,
+            'is_active': None,
+            'status': None,
+            'created_at': None
+        }
+        doc = TypesenseService.prepare_product_doc(raw_product)
+        self.assertEqual(doc['id'], '101')
+        self.assertEqual(doc['sku'], 'TRDX-TEST-001')
+        self.assertEqual(doc['title'], 'Test Silk Saree')
+        self.assertEqual(doc['description'], '')
+        self.assertEqual(doc['category'], 'Uncategorized')
+        self.assertEqual(doc['supplier_id'], 0)
+        self.assertEqual(doc['price'], 0.0)
+        self.assertEqual(doc['compare_at_price'], 0.0)
+        self.assertEqual(doc['stock'], 0)
+        self.assertFalse(doc['is_gi_tagged'])
+        self.assertTrue(doc['is_active'])
+        self.assertEqual(doc['status'], 'active')
+        self.assertEqual(doc['created_at'], 0)
+
+
+class SearchAndIndexApiTests(TestCase):
+    def setUp(self):
+        self.client = APIClient()
+
+    @patch.object(TypesenseService, 'search_products')
+    def test_search_view_defaults(self, mock_search):
+        mock_search.return_value = {
+            'hits': [{'document': {'id': '1', 'title': 'Product 1'}}],
+            'found': 1,
+            'facet_counts': []
+        }
+
+        response = self.client.get('/api/search/', {'q': 'saree'})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 1)
+        self.assertEqual(len(response.data['results']), 1)
+        mock_search.assert_called_once()
+        _, kwargs = mock_search.call_args
+        self.assertEqual(kwargs['query'], 'saree')
+        self.assertIn('is_active:=true', kwargs['filter_by'])
+        self.assertEqual(kwargs['sort_by'], '_text_match:desc,created_at:desc')
+
+    @patch.object(TypesenseService, 'search_products')
+    def test_search_view_filters_and_sorting(self, mock_search):
+        mock_search.return_value = {'hits': [], 'found': 0, 'facet_counts': []}
+
+        response = self.client.get('/api/search/', {
+            'q': '*',
+            'category': 'Textiles & Silk',
+            'min_price': '100',
+            'max_price': '500',
+            'is_gi_tagged': 'false',
+            'supplier_id': '42',
+            'sort_by': 'price_asc',
+            'page': '2',
+            'per_page': '10'
+        })
+        self.assertEqual(response.status_code, 200)
+        _, kwargs = mock_search.call_args
+        self.assertIn('category:="Textiles & Silk"', kwargs['filter_by'])
+        self.assertIn('price:[100.0..500.0]', kwargs['filter_by'])
+        self.assertIn('is_gi_tagged:=false', kwargs['filter_by'])
+        self.assertIn('supplier_id:=42', kwargs['filter_by'])
+        self.assertEqual(kwargs['sort_by'], 'price:asc')
+        self.assertEqual(kwargs['page'], 2)
+        self.assertEqual(kwargs['per_page'], 10)
+
+    @patch.object(TypesenseService, 'index_product')
+    def test_index_single_product(self, mock_index):
+        mock_index.return_value = {'success': True}
+        payload = {'id': 1, 'sku': 'SKU123', 'title': 'Test Item', 'price': 99.0}
+        response = self.client.post('/api/index/product/', payload, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['status'], 'indexed')
+
+    @patch.object(TypesenseService, 'index_products_batch')
+    def test_index_batch_products(self, mock_batch):
+        mock_batch.return_value = [{'success': True}, {'success': True}]
+        payload = [
+            {'id': 1, 'sku': 'SKU1', 'title': 'Item 1'},
+            {'id': 2, 'sku': 'SKU2', 'title': 'Item 2'}
+        ]
+        response = self.client.post('/api/index/product/', payload, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 2)
+
+    @patch.object(TypesenseService, 'delete_product')
+    def test_delete_product(self, mock_delete):
+        mock_delete.return_value = {'deleted': True}
+        response = self.client.delete('/api/index/product/TRDX-001/')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['status'], 'deleted')
+        mock_delete.assert_called_once_with('TRDX-001')

api/typesense_client.py

@@ -1,4 +1,4 @@
-import typesense
+import typesense
 from django.conf import settings
 import logging
 
@@ -26,8 +26,42 @@ PRODUCTS_SCHEMA = {
     'default_sorting_field': 'created_at'
 }
 
+def safe_str(val, default=""):
+    if val is None:
+        return default
+    return str(val)
+
+def safe_int(val, default=0):
+    if val is None or val == "":
+        return default
+    try:
+        return int(val)
+    except (ValueError, TypeError):
+        return default
+
+def safe_float(val, default=0.0):
+    if val is None or val == "":
+        return default
+    try:
+        return float(val)
+    except (ValueError, TypeError):
+        return default
+
+def safe_bool(val, default=False):
+    if val is None:
+        return default
+    if isinstance(val, bool):
+        return val
+    if isinstance(val, (int, float)):
+        return bool(val)
+    if isinstance(val, str):
+        return val.lower() in ("true", "1", "yes")
+    return default
+
+
 class TypesenseService:
     _client = None
+    _schema_initialized = False
 
     @classmethod
     def get_client(cls):
@@ -45,53 +79,81 @@ class TypesenseService:
         return cls._client
 
     @classmethod
-    def ensure_schema(cls):
+    def ensure_schema(cls, force=False):
+        if cls._schema_initialized and not force:
+            return
         client = cls.get_client()
         try:
             client.collections['products'].retrieve()
             logger.info("Typesense 'products' collection already exists.")
+            cls._schema_initialized = True
         except Exception:
             logger.info("Creating Typesense 'products' collection schema...")
             client.collections.create(PRODUCTS_SCHEMA)
+            cls._schema_initialized = True
+
+    @classmethod
+    def prepare_product_doc(cls, product_dict):
+        doc_id = product_dict.get('id')
+        if doc_id is None:
+            doc_id = product_dict.get('sku')
+
+        return {
+            'id': safe_str(doc_id),
+            'title': safe_str(product_dict.get('title')),
+            'description': safe_str(product_dict.get('description')),
+            'category': safe_str(product_dict.get('category'), 'Uncategorized'),
+            'supplier_id': safe_int(product_dict.get('supplier_id'), 0),
+            'supplier_name': safe_str(product_dict.get('supplier_name')),
+            'price': safe_float(product_dict.get('price'), 0.0),
+            'compare_at_price': safe_float(product_dict.get('compare_at_price'), 0.0),
+            'stock': safe_int(product_dict.get('stock'), 0),
+            'sku': safe_str(product_dict.get('sku')),
+            'image': safe_str(product_dict.get('image')),
+            'is_gi_tagged': safe_bool(product_dict.get('is_gi_tagged'), False),
+            'is_active': safe_bool(product_dict.get('is_active'), True),
+            'status': safe_str(product_dict.get('status'), 'active'),
+            'created_at': safe_int(product_dict.get('created_at'), 0)
+        }
 
     @classmethod
     def index_product(cls, product_dict):
         cls.ensure_schema()
         client = cls.get_client()
-        doc = {
-            'id': str(product_dict.get('id', product_dict.get('sku'))),
-            'title': str(product_dict.get('title', '')),
-            'description': str(product_dict.get('description', '') or ''),
-            'category': str(product_dict.get('category', 'Uncategorized')),
-            'supplier_id': int(product_dict.get('supplier_id', 0)),
-            'supplier_name': str(product_dict.get('supplier_name', '') or ''),
-            'price': float(product_dict.get('price', 0.0)),
-            'compare_at_price': float(product_dict.get('compare_at_price', 0.0) or 0.0),
-            'stock': int(product_dict.get('stock', 0)),
-            'sku': str(product_dict.get('sku', '')),
-            'image': str(product_dict.get('image', '') or ''),
-            'is_gi_tagged': bool(product_dict.get('is_gi_tagged', False)),
-            'is_active': bool(product_dict.get('is_active', True)),
-            'status': str(product_dict.get('status', 'active')),
-            'created_at': int(product_dict.get('created_at', 0))
-        }
+        doc = cls.prepare_product_doc(product_dict)
         return client.collections['products'].documents.upsert(doc)
 
+    @classmethod
+    def index_products_batch(cls, products_list):
+        if not products_list:
+            return []
+        cls.ensure_schema()
+        client = cls.get_client()
+        docs = [cls.prepare_product_doc(p) for p in products_list]
+        return client.collections['products'].documents.import_(docs, {'action': 'upsert'})
+
     @classmethod
     def delete_product(cls, product_id):
         client = cls.get_client()
+        pid_str = str(product_id).strip()
         try:
-            return client.collections['products'].documents[str(product_id)].delete()
+            return client.collections['products'].documents[pid_str].delete()
         except Exception as e:
-            logger.warning(f"Could not delete document {product_id} from Typesense: {e}")
-            return None
+            logger.info(f"Direct delete by ID {pid_str} failed ({e}), attempting delete by SKU filter...")
+            try:
+                return client.collections['products'].documents.delete({'filter_by': f'sku:="{pid_str}"'})
+            except Exception as e_filter:
+                logger.warning(f"Could not delete document with identifier {pid_str} from Typesense: {e_filter}")
+                return None
 
     @classmethod
     def search_products(cls, query="*", filter_by="", sort_by="", page=1, per_page=20):
         cls.ensure_schema()
         client = cls.get_client()
+        q_clean = query.strip() if query else "*"
+
         search_parameters = {
-            'q': query if query else '*',
+            'q': q_clean if q_clean else '*',
             'query_by': 'title,description,category,sku',
             'facet_by': 'category,is_gi_tagged,supplier_id',
             'page': page,
@@ -99,11 +161,16 @@ class TypesenseService:
             'num_typos': 2,
             'prefix': True
         }
+
         if filter_by:
             search_parameters['filter_by'] = filter_by
+
         if sort_by:
             search_parameters['sort_by'] = sort_by
         else:
-            search_parameters['sort_by'] = 'created_at:desc'
+            if q_clean and q_clean != '*':
+                search_parameters['sort_by'] = '_text_match:desc,created_at:desc'
+            else:
+                search_parameters['sort_by'] = 'created_at:desc'
 
         return client.collections['products'].documents.search(search_parameters)

api/views.py

@@ -1,4 +1,4 @@
-from rest_framework.views import APIView
+from rest_framework.views import APIView
 from rest_framework.response import Response
 from rest_framework import status
 from django.db import connection
@@ -17,33 +17,56 @@ class ProductSearchView(APIView):
             max_price = request.query_params.get('max_price')
             is_gi_tagged = request.query_params.get('is_gi_tagged')
             supplier_id = request.query_params.get('supplier_id')
-            sort_by_param = request.query_params.get('sort_by', 'newest')
-            page = int(request.query_params.get('page', 1))
-            per_page = int(request.query_params.get('per_page', 20))
+            sort_by_param = request.query_params.get('sort_by', '').strip()
+
+            try:
+                page = max(1, int(request.query_params.get('page', 1)))
+                per_page = min(250, max(1, int(request.query_params.get('per_page', 20))))
+            except (ValueError, TypeError):
+                return Response({'error': 'Invalid page or per_page parameter'}, status=status.HTTP_400_BAD_REQUEST)
 
             filter_clauses = ['is_active:=true']
 
             if category:
-                filter_clauses.append(f'category:="{category}"')
+                escaped_category = category.replace('"', '\\"')
+                filter_clauses.append(f'category:="{escaped_category}"')
+
             if supplier_id:
-                filter_clauses.append(f'supplier_id:={supplier_id}')
-            if is_gi_tagged is not None and is_gi_tagged.lower() in ('true', '1'):
-                filter_clauses.append('is_gi_tagged:=true')
-            
+                try:
+                    s_id = int(supplier_id)
+                    filter_clauses.append(f'supplier_id:={s_id}')
+                except (ValueError, TypeError):
+                    return Response({'error': 'supplier_id must be an integer'}, status=status.HTTP_400_BAD_REQUEST)
+
+            if is_gi_tagged is not None:
+                val = is_gi_tagged.strip().lower()
+                if val in ('true', '1'):
+                    filter_clauses.append('is_gi_tagged:=true')
+                elif val in ('false', '0'):
+                    filter_clauses.append('is_gi_tagged:=false')
+
             if min_price is not None or max_price is not None:
-                p_min = float(min_price) if min_price else 0.0
-                p_max = float(max_price) if max_price else 9999999.0
-                filter_clauses.append(f'price:[{p_min}..{p_max}]')
+                try:
+                    p_min = float(min_price) if min_price else 0.0
+                    p_max = float(max_price) if max_price else 9999999.0
+                    if p_min > p_max:
+                        p_min, p_max = p_max, p_min
+                    filter_clauses.append(f'price:[{p_min}..{p_max}]')
+                except (ValueError, TypeError):
+                    return Response({'error': 'Price filters must be valid numbers'}, status=status.HTTP_400_BAD_REQUEST)
 
             filter_by_str = ' && '.join(filter_clauses)
 
-            sort_by_str = 'created_at:desc'
             if sort_by_param == 'price_asc':
                 sort_by_str = 'price:asc'
             elif sort_by_param == 'price_desc':
                 sort_by_str = 'price:desc'
-            elif sort_by_param == 'relevance' and q != '*':
-                sort_by_str = ''
+            elif sort_by_param == 'relevance':
+                sort_by_str = '_text_match:desc,created_at:desc'
+            elif sort_by_param == 'newest':
+                sort_by_str = 'created_at:desc'
+            else:
+                sort_by_str = '_text_match:desc,created_at:desc' if (q and q != '*') else 'created_at:desc'
 
             res = TypesenseService.search_products(
                 query=q,
@@ -76,12 +99,26 @@ class ProductSearchView(APIView):
 class IndexProductView(APIView):
     def post(self, request):
         try:
-            product_data = request.data
-            if not product_data or ('id' not in product_data and 'sku' not in product_data):
-                return Response({'error': 'Product id or sku is required'}, status=status.HTTP_400_BAD_REQUEST)
-            
-            res = TypesenseService.index_product(product_data)
-            return Response({'status': 'indexed', 'result': res}, status=status.HTTP_200_OK)
+            data = request.data
+            if not data:
+                return Response({'error': 'Payload cannot be empty'}, status=status.HTTP_400_BAD_REQUEST)
+
+            if isinstance(data, list):
+                valid_items = []
+                for item in data:
+                    if isinstance(item, dict) and ('id' in item or 'sku' in item):
+                        valid_items.append(item)
+                if not valid_items:
+                    return Response({'error': 'Each product in list must have an id or sku'}, status=status.HTTP_400_BAD_REQUEST)
+                res = TypesenseService.index_products_batch(valid_items)
+                return Response({'status': 'indexed', 'count': len(valid_items), 'result': res}, status=status.HTTP_200_OK)
+            elif isinstance(data, dict):
+                if 'id' not in data and 'sku' not in data:
+                    return Response({'error': 'Product id or sku is required'}, status=status.HTTP_400_BAD_REQUEST)
+                res = TypesenseService.index_product(data)
+                return Response({'status': 'indexed', 'result': res}, status=status.HTTP_200_OK)
+            else:
+                return Response({'error': 'Invalid JSON format. Expected JSON object or array.'}, status=status.HTTP_400_BAD_REQUEST)
         except Exception as e:
             logger.error(f"Product Index Error: {e}", exc_info=True)
             return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -90,6 +127,8 @@ class IndexProductView(APIView):
 class DeleteProductView(APIView):
     def delete(self, request, product_id):
         try:
+            if not product_id or not str(product_id).strip():
+                return Response({'error': 'Product identifier is required'}, status=status.HTTP_400_BAD_REQUEST)
             res = TypesenseService.delete_product(product_id)
             return Response({'status': 'deleted', 'result': res}, status=status.HTTP_200_OK)
         except Exception as e:
@@ -100,9 +139,11 @@ class DeleteProductView(APIView):
 class ReindexCatalogView(APIView):
     def post(self, request):
         try:
+            TypesenseService.ensure_schema(force=True)
             indexed_count = 0
-            TypesenseService.ensure_schema()
-            
+            batch_size = 250
+            batch = []
+
             with connection.cursor() as cursor:
                 cursor.execute("""
                     SELECT id, sku, title, description, category, supplier_id, price, 
@@ -111,11 +152,22 @@ class ReindexCatalogView(APIView):
                 """)
                 columns = [col[0] for col in cursor.description]
                 rows = cursor.fetchall()
-                for row in rows:
+                now_ts = int(time.time())
+
+                for idx, row in enumerate(rows):
                     item = dict(zip(columns, row))
-                    item['created_at'] = int(time.time())
-                    TypesenseService.index_product(item)
-                    indexed_count += 1
+                    # Retain descending ordering with timestamp offset
+                    item['created_at'] = now_ts + idx
+                    batch.append(item)
+
+                    if len(batch) >= batch_size:
+                        TypesenseService.index_products_batch(batch)
+                        indexed_count += len(batch)
+                        batch = []
+
+                if batch:
+                    TypesenseService.index_products_batch(batch)
+                    indexed_count += len(batch)
 
             return Response({'status': 'completed', 'total_indexed': indexed_count}, status=status.HTTP_200_OK)
         except Exception as e:

logs/.gitkeep

@@ -0,0 +1 @@
+# Keep logs directory tracked in version control