diff --git a/.forgejo/workflows/beta.yml b/.forgejo/workflows/beta.yml index ed71c5d..e60d5bc 100644 --- a/.forgejo/workflows/beta.yml +++ b/.forgejo/workflows/beta.yml @@ -1,4 +1,4 @@ -name: Deploy Beta (NATIVE) +name: Deploy Beta (NATIVE) on: push: @@ -118,6 +118,8 @@ jobs: sudo mkdir -p /home/ubuntu/searchengine sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine tar -xzf /tmp/app.tar.gz -C /home/ubuntu/searchengine + mkdir -p /home/ubuntu/searchengine/logs + sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine/logs # ── 4. Set up python virtual environment & dependencies ──────── cd /home/ubuntu/searchengine @@ -125,14 +127,22 @@ jobs: source venv/bin/activate pip install --upgrade pip pip install -r requirements.txt + python manage.py migrate --noinput # ── 5. Systemd Service Deployment ────────────────────────────── sudo cp /tmp/searchengine.service /etc/systemd/system/searchengine.service sudo systemctl daemon-reload sudo systemctl enable searchengine.service - sudo systemctl restart searchengine.service + sudo systemctl restart searchengine.service || { + echo "searchengine.service failed to restart! Checking journalctl:" + sudo journalctl -u searchengine.service -n 50 --no-pager + exit 1 + } # ── 6. Nginx Config ─────────────────────────────────────────── + if [ -f "/etc/letsencrypt/live/betasearch-api.tradhox.com/fullchain.pem" ]; then + sed -i 's|/etc/letsencrypt/live/betapartner-api.tradhox.com|/etc/letsencrypt/live/betasearch-api.tradhox.com|g' /tmp/nginx-proxy.conf + fi sudo cp /tmp/nginx-proxy.conf /etc/nginx/sites-available/searchengine sudo ln -sf /etc/nginx/sites-available/searchengine /etc/nginx/sites-enabled/searchengine sudo nginx -t && sudo systemctl reload nginx diff --git a/api/management/commands/reindex_typesense.py b/api/management/commands/reindex_typesense.py index de87108..210fcce 100644 --- a/api/management/commands/reindex_typesense.py +++ b/api/management/commands/reindex_typesense.py @@ -1,4 +1,4 @@ -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand from django.db import connection from api.typesense_client import TypesenseService import time @@ -6,11 +6,24 @@ import time class Command(BaseCommand): help = "Reindex all products from PostgreSQL database into Typesense search index." - def handle(self, *args, **options): - self.stdout.write(self.style.SUCCESS("Starting Typesense reindex...")) - TypesenseService.ensure_schema() - count = 0 + def add_arguments(self, parser): + parser.add_argument( + '--batch-size', + type=int, + default=250, + help='Number of records to index per batch request.' + ) + def handle(self, *args, **options): + batch_size = options.get('batch_size', 250) + self.stdout.write(self.style.SUCCESS("Ensuring Typesense schema...")) + TypesenseService.ensure_schema(force=True) + + count = 0 + batch = [] + now_ts = int(time.time()) + + self.stdout.write(self.style.SUCCESS("Fetching products from PostgreSQL (api_product)...")) with connection.cursor() as cursor: cursor.execute(""" SELECT id, sku, title, description, category, supplier_id, price, @@ -19,11 +32,23 @@ class Command(BaseCommand): """) columns = [col[0] for col in cursor.description] rows = cursor.fetchall() - for row in rows: + total_rows = len(rows) + self.stdout.write(f"Found {total_rows} products to index.") + + for idx, row in enumerate(rows): item = dict(zip(columns, row)) - item['created_at'] = int(time.time()) - TypesenseService.index_product(item) - count += 1 - self.stdout.write(f"Indexed product SKU: {item.get('sku')} | Title: {item.get('title')}") + item['created_at'] = now_ts + idx + batch.append(item) + + if len(batch) >= batch_size: + TypesenseService.index_products_batch(batch) + count += len(batch) + self.stdout.write(f"Indexed batch: {count}/{total_rows} products...") + batch = [] + + if batch: + TypesenseService.index_products_batch(batch) + count += len(batch) + self.stdout.write(f"Indexed final batch: {count}/{total_rows} products...") self.stdout.write(self.style.SUCCESS(f"Successfully reindexed {count} products into Typesense.")) diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py new file mode 100644 index 0000000..4906a2e --- /dev/null +++ b/api/migrations/0001_initial.py @@ -0,0 +1,24 @@ +# Generated by Django 6.1 on 2026-10-15 14:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ProductIndexLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sku', models.CharField(max_length=100, unique=True)), + ('status', models.CharField(default='synced', max_length=50)), + ('last_synced_at', models.DateTimeField(auto_now=True)), + ('error', models.TextField(blank=True, null=True)), + ], + ), + ] diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/tests.py b/api/tests.py new file mode 100644 index 0000000..534ea14 --- /dev/null +++ b/api/tests.py @@ -0,0 +1,134 @@ +from django.test import TestCase +from rest_framework.test import APIClient +from unittest.mock import patch, MagicMock +from api.typesense_client import ( + TypesenseService, + safe_str, + safe_int, + safe_float, + safe_bool, +) + +class TypesenseServiceUnitTests(TestCase): + def test_safe_helpers(self): + self.assertEqual(safe_str(None, "default"), "default") + self.assertEqual(safe_str(123), "123") + self.assertEqual(safe_int(None, 10), 10) + self.assertEqual(safe_int("42"), 42) + self.assertEqual(safe_int("invalid", 5), 5) + self.assertEqual(safe_float(None, 3.14), 3.14) + self.assertEqual(safe_float("99.95"), 99.95) + self.assertEqual(safe_float("invalid", 0.0), 0.0) + self.assertTrue(safe_bool("true")) + self.assertTrue(safe_bool("1")) + self.assertFalse(safe_bool("false")) + self.assertFalse(safe_bool("0")) + self.assertFalse(safe_bool(None, False)) + + def test_prepare_product_doc_handles_nulls_gracefully(self): + raw_product = { + 'id': 101, + 'sku': 'TRDX-TEST-001', + 'title': 'Test Silk Saree', + 'description': None, + 'category': None, + 'supplier_id': None, + 'supplier_name': None, + 'price': None, + 'compare_at_price': None, + 'stock': None, + 'image': None, + 'is_gi_tagged': None, + 'is_active': None, + 'status': None, + 'created_at': None + } + doc = TypesenseService.prepare_product_doc(raw_product) + self.assertEqual(doc['id'], '101') + self.assertEqual(doc['sku'], 'TRDX-TEST-001') + self.assertEqual(doc['title'], 'Test Silk Saree') + self.assertEqual(doc['description'], '') + self.assertEqual(doc['category'], 'Uncategorized') + self.assertEqual(doc['supplier_id'], 0) + self.assertEqual(doc['price'], 0.0) + self.assertEqual(doc['compare_at_price'], 0.0) + self.assertEqual(doc['stock'], 0) + self.assertFalse(doc['is_gi_tagged']) + self.assertTrue(doc['is_active']) + self.assertEqual(doc['status'], 'active') + self.assertEqual(doc['created_at'], 0) + + +class SearchAndIndexApiTests(TestCase): + def setUp(self): + self.client = APIClient() + + @patch.object(TypesenseService, 'search_products') + def test_search_view_defaults(self, mock_search): + mock_search.return_value = { + 'hits': [{'document': {'id': '1', 'title': 'Product 1'}}], + 'found': 1, + 'facet_counts': [] + } + + response = self.client.get('/api/search/', {'q': 'saree'}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 1) + self.assertEqual(len(response.data['results']), 1) + mock_search.assert_called_once() + _, kwargs = mock_search.call_args + self.assertEqual(kwargs['query'], 'saree') + self.assertIn('is_active:=true', kwargs['filter_by']) + self.assertEqual(kwargs['sort_by'], '_text_match:desc,created_at:desc') + + @patch.object(TypesenseService, 'search_products') + def test_search_view_filters_and_sorting(self, mock_search): + mock_search.return_value = {'hits': [], 'found': 0, 'facet_counts': []} + + response = self.client.get('/api/search/', { + 'q': '*', + 'category': 'Textiles & Silk', + 'min_price': '100', + 'max_price': '500', + 'is_gi_tagged': 'false', + 'supplier_id': '42', + 'sort_by': 'price_asc', + 'page': '2', + 'per_page': '10' + }) + self.assertEqual(response.status_code, 200) + _, kwargs = mock_search.call_args + self.assertIn('category:="Textiles & Silk"', kwargs['filter_by']) + self.assertIn('price:[100.0..500.0]', kwargs['filter_by']) + self.assertIn('is_gi_tagged:=false', kwargs['filter_by']) + self.assertIn('supplier_id:=42', kwargs['filter_by']) + self.assertEqual(kwargs['sort_by'], 'price:asc') + self.assertEqual(kwargs['page'], 2) + self.assertEqual(kwargs['per_page'], 10) + + @patch.object(TypesenseService, 'index_product') + def test_index_single_product(self, mock_index): + mock_index.return_value = {'success': True} + payload = {'id': 1, 'sku': 'SKU123', 'title': 'Test Item', 'price': 99.0} + response = self.client.post('/api/index/product/', payload, format='json') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['status'], 'indexed') + + @patch.object(TypesenseService, 'index_products_batch') + def test_index_batch_products(self, mock_batch): + mock_batch.return_value = [{'success': True}, {'success': True}] + payload = [ + {'id': 1, 'sku': 'SKU1', 'title': 'Item 1'}, + {'id': 2, 'sku': 'SKU2', 'title': 'Item 2'} + ] + response = self.client.post('/api/index/product/', payload, format='json') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 2) + + @patch.object(TypesenseService, 'delete_product') + def test_delete_product(self, mock_delete): + mock_delete.return_value = {'deleted': True} + response = self.client.delete('/api/index/product/TRDX-001/') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['status'], 'deleted') + mock_delete.assert_called_once_with('TRDX-001') diff --git a/api/typesense_client.py b/api/typesense_client.py index 2864059..bccce83 100644 --- a/api/typesense_client.py +++ b/api/typesense_client.py @@ -1,4 +1,4 @@ -import typesense +import typesense from django.conf import settings import logging @@ -26,8 +26,42 @@ PRODUCTS_SCHEMA = { 'default_sorting_field': 'created_at' } +def safe_str(val, default=""): + if val is None: + return default + return str(val) + +def safe_int(val, default=0): + if val is None or val == "": + return default + try: + return int(val) + except (ValueError, TypeError): + return default + +def safe_float(val, default=0.0): + if val is None or val == "": + return default + try: + return float(val) + except (ValueError, TypeError): + return default + +def safe_bool(val, default=False): + if val is None: + return default + if isinstance(val, bool): + return val + if isinstance(val, (int, float)): + return bool(val) + if isinstance(val, str): + return val.lower() in ("true", "1", "yes") + return default + + class TypesenseService: _client = None + _schema_initialized = False @classmethod def get_client(cls): @@ -45,53 +79,81 @@ class TypesenseService: return cls._client @classmethod - def ensure_schema(cls): + def ensure_schema(cls, force=False): + if cls._schema_initialized and not force: + return client = cls.get_client() try: client.collections['products'].retrieve() logger.info("Typesense 'products' collection already exists.") + cls._schema_initialized = True except Exception: logger.info("Creating Typesense 'products' collection schema...") client.collections.create(PRODUCTS_SCHEMA) + cls._schema_initialized = True + + @classmethod + def prepare_product_doc(cls, product_dict): + doc_id = product_dict.get('id') + if doc_id is None: + doc_id = product_dict.get('sku') + + return { + 'id': safe_str(doc_id), + 'title': safe_str(product_dict.get('title')), + 'description': safe_str(product_dict.get('description')), + 'category': safe_str(product_dict.get('category'), 'Uncategorized'), + 'supplier_id': safe_int(product_dict.get('supplier_id'), 0), + 'supplier_name': safe_str(product_dict.get('supplier_name')), + 'price': safe_float(product_dict.get('price'), 0.0), + 'compare_at_price': safe_float(product_dict.get('compare_at_price'), 0.0), + 'stock': safe_int(product_dict.get('stock'), 0), + 'sku': safe_str(product_dict.get('sku')), + 'image': safe_str(product_dict.get('image')), + 'is_gi_tagged': safe_bool(product_dict.get('is_gi_tagged'), False), + 'is_active': safe_bool(product_dict.get('is_active'), True), + 'status': safe_str(product_dict.get('status'), 'active'), + 'created_at': safe_int(product_dict.get('created_at'), 0) + } @classmethod def index_product(cls, product_dict): cls.ensure_schema() client = cls.get_client() - doc = { - 'id': str(product_dict.get('id', product_dict.get('sku'))), - 'title': str(product_dict.get('title', '')), - 'description': str(product_dict.get('description', '') or ''), - 'category': str(product_dict.get('category', 'Uncategorized')), - 'supplier_id': int(product_dict.get('supplier_id', 0)), - 'supplier_name': str(product_dict.get('supplier_name', '') or ''), - 'price': float(product_dict.get('price', 0.0)), - 'compare_at_price': float(product_dict.get('compare_at_price', 0.0) or 0.0), - 'stock': int(product_dict.get('stock', 0)), - 'sku': str(product_dict.get('sku', '')), - 'image': str(product_dict.get('image', '') or ''), - 'is_gi_tagged': bool(product_dict.get('is_gi_tagged', False)), - 'is_active': bool(product_dict.get('is_active', True)), - 'status': str(product_dict.get('status', 'active')), - 'created_at': int(product_dict.get('created_at', 0)) - } + doc = cls.prepare_product_doc(product_dict) return client.collections['products'].documents.upsert(doc) + @classmethod + def index_products_batch(cls, products_list): + if not products_list: + return [] + cls.ensure_schema() + client = cls.get_client() + docs = [cls.prepare_product_doc(p) for p in products_list] + return client.collections['products'].documents.import_(docs, {'action': 'upsert'}) + @classmethod def delete_product(cls, product_id): client = cls.get_client() + pid_str = str(product_id).strip() try: - return client.collections['products'].documents[str(product_id)].delete() + return client.collections['products'].documents[pid_str].delete() except Exception as e: - logger.warning(f"Could not delete document {product_id} from Typesense: {e}") - return None + logger.info(f"Direct delete by ID {pid_str} failed ({e}), attempting delete by SKU filter...") + try: + return client.collections['products'].documents.delete({'filter_by': f'sku:="{pid_str}"'}) + except Exception as e_filter: + logger.warning(f"Could not delete document with identifier {pid_str} from Typesense: {e_filter}") + return None @classmethod def search_products(cls, query="*", filter_by="", sort_by="", page=1, per_page=20): cls.ensure_schema() client = cls.get_client() + q_clean = query.strip() if query else "*" + search_parameters = { - 'q': query if query else '*', + 'q': q_clean if q_clean else '*', 'query_by': 'title,description,category,sku', 'facet_by': 'category,is_gi_tagged,supplier_id', 'page': page, @@ -99,11 +161,16 @@ class TypesenseService: 'num_typos': 2, 'prefix': True } + if filter_by: search_parameters['filter_by'] = filter_by + if sort_by: search_parameters['sort_by'] = sort_by else: - search_parameters['sort_by'] = 'created_at:desc' + if q_clean and q_clean != '*': + search_parameters['sort_by'] = '_text_match:desc,created_at:desc' + else: + search_parameters['sort_by'] = 'created_at:desc' return client.collections['products'].documents.search(search_parameters) diff --git a/api/views.py b/api/views.py index 90aa2b7..8a1c394 100644 --- a/api/views.py +++ b/api/views.py @@ -1,4 +1,4 @@ -from rest_framework.views import APIView +from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import status from django.db import connection @@ -17,33 +17,56 @@ class ProductSearchView(APIView): max_price = request.query_params.get('max_price') is_gi_tagged = request.query_params.get('is_gi_tagged') supplier_id = request.query_params.get('supplier_id') - sort_by_param = request.query_params.get('sort_by', 'newest') - page = int(request.query_params.get('page', 1)) - per_page = int(request.query_params.get('per_page', 20)) + sort_by_param = request.query_params.get('sort_by', '').strip() + + try: + page = max(1, int(request.query_params.get('page', 1))) + per_page = min(250, max(1, int(request.query_params.get('per_page', 20)))) + except (ValueError, TypeError): + return Response({'error': 'Invalid page or per_page parameter'}, status=status.HTTP_400_BAD_REQUEST) filter_clauses = ['is_active:=true'] if category: - filter_clauses.append(f'category:="{category}"') + escaped_category = category.replace('"', '\\"') + filter_clauses.append(f'category:="{escaped_category}"') + if supplier_id: - filter_clauses.append(f'supplier_id:={supplier_id}') - if is_gi_tagged is not None and is_gi_tagged.lower() in ('true', '1'): - filter_clauses.append('is_gi_tagged:=true') - + try: + s_id = int(supplier_id) + filter_clauses.append(f'supplier_id:={s_id}') + except (ValueError, TypeError): + return Response({'error': 'supplier_id must be an integer'}, status=status.HTTP_400_BAD_REQUEST) + + if is_gi_tagged is not None: + val = is_gi_tagged.strip().lower() + if val in ('true', '1'): + filter_clauses.append('is_gi_tagged:=true') + elif val in ('false', '0'): + filter_clauses.append('is_gi_tagged:=false') + if min_price is not None or max_price is not None: - p_min = float(min_price) if min_price else 0.0 - p_max = float(max_price) if max_price else 9999999.0 - filter_clauses.append(f'price:[{p_min}..{p_max}]') + try: + p_min = float(min_price) if min_price else 0.0 + p_max = float(max_price) if max_price else 9999999.0 + if p_min > p_max: + p_min, p_max = p_max, p_min + filter_clauses.append(f'price:[{p_min}..{p_max}]') + except (ValueError, TypeError): + return Response({'error': 'Price filters must be valid numbers'}, status=status.HTTP_400_BAD_REQUEST) filter_by_str = ' && '.join(filter_clauses) - sort_by_str = 'created_at:desc' if sort_by_param == 'price_asc': sort_by_str = 'price:asc' elif sort_by_param == 'price_desc': sort_by_str = 'price:desc' - elif sort_by_param == 'relevance' and q != '*': - sort_by_str = '' + elif sort_by_param == 'relevance': + sort_by_str = '_text_match:desc,created_at:desc' + elif sort_by_param == 'newest': + sort_by_str = 'created_at:desc' + else: + sort_by_str = '_text_match:desc,created_at:desc' if (q and q != '*') else 'created_at:desc' res = TypesenseService.search_products( query=q, @@ -76,12 +99,26 @@ class ProductSearchView(APIView): class IndexProductView(APIView): def post(self, request): try: - product_data = request.data - if not product_data or ('id' not in product_data and 'sku' not in product_data): - return Response({'error': 'Product id or sku is required'}, status=status.HTTP_400_BAD_REQUEST) - - res = TypesenseService.index_product(product_data) - return Response({'status': 'indexed', 'result': res}, status=status.HTTP_200_OK) + data = request.data + if not data: + return Response({'error': 'Payload cannot be empty'}, status=status.HTTP_400_BAD_REQUEST) + + if isinstance(data, list): + valid_items = [] + for item in data: + if isinstance(item, dict) and ('id' in item or 'sku' in item): + valid_items.append(item) + if not valid_items: + return Response({'error': 'Each product in list must have an id or sku'}, status=status.HTTP_400_BAD_REQUEST) + res = TypesenseService.index_products_batch(valid_items) + return Response({'status': 'indexed', 'count': len(valid_items), 'result': res}, status=status.HTTP_200_OK) + elif isinstance(data, dict): + if 'id' not in data and 'sku' not in data: + return Response({'error': 'Product id or sku is required'}, status=status.HTTP_400_BAD_REQUEST) + res = TypesenseService.index_product(data) + return Response({'status': 'indexed', 'result': res}, status=status.HTTP_200_OK) + else: + return Response({'error': 'Invalid JSON format. Expected JSON object or array.'}, status=status.HTTP_400_BAD_REQUEST) except Exception as e: logger.error(f"Product Index Error: {e}", exc_info=True) return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) @@ -90,6 +127,8 @@ class IndexProductView(APIView): class DeleteProductView(APIView): def delete(self, request, product_id): try: + if not product_id or not str(product_id).strip(): + return Response({'error': 'Product identifier is required'}, status=status.HTTP_400_BAD_REQUEST) res = TypesenseService.delete_product(product_id) return Response({'status': 'deleted', 'result': res}, status=status.HTTP_200_OK) except Exception as e: @@ -100,9 +139,11 @@ class DeleteProductView(APIView): class ReindexCatalogView(APIView): def post(self, request): try: + TypesenseService.ensure_schema(force=True) indexed_count = 0 - TypesenseService.ensure_schema() - + batch_size = 250 + batch = [] + with connection.cursor() as cursor: cursor.execute(""" SELECT id, sku, title, description, category, supplier_id, price, @@ -111,11 +152,22 @@ class ReindexCatalogView(APIView): """) columns = [col[0] for col in cursor.description] rows = cursor.fetchall() - for row in rows: + now_ts = int(time.time()) + + for idx, row in enumerate(rows): item = dict(zip(columns, row)) - item['created_at'] = int(time.time()) - TypesenseService.index_product(item) - indexed_count += 1 + # Retain descending ordering with timestamp offset + item['created_at'] = now_ts + idx + batch.append(item) + + if len(batch) >= batch_size: + TypesenseService.index_products_batch(batch) + indexed_count += len(batch) + batch = [] + + if batch: + TypesenseService.index_products_batch(batch) + indexed_count += len(batch) return Response({'status': 'completed', 'total_indexed': indexed_count}, status=status.HTTP_200_OK) except Exception as e: diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..6f6eb9d --- /dev/null +++ b/logs/.gitkeep @@ -0,0 +1 @@ +# Keep logs directory tracked in version control