commit bba11555f0484428bb7836f8a1d7696e4daa442b Author: vignesh Date: Sun Oct 18 15:13:03 2026 +0000 Initial commit diff --git a/.forgejo/workflows/beta.yml b/.forgejo/workflows/beta.yml new file mode 100644 index 0000000..e60d5bc --- /dev/null +++ b/.forgejo/workflows/beta.yml @@ -0,0 +1,156 @@ +name: Deploy Beta (NATIVE) + +on: + push: + branches: + - beta + +jobs: + deploy: + runs-on: ubuntu-latest + container: + image: node:22-alpine + steps: + - name: Install git + run: apk add --no-cache git + + - name: Checkout Code + uses: https://github.com/actions/checkout@v4 + + - name: Generate CloudWatch and Nginx Configs Locally + run: | + mkdir -p tmp + cat << 'EOF' > tmp/amazon-cloudwatch-agent.json + { + "agent": { "metrics_collection_interval": 60, "run_as_user": "root" }, + "metrics": { + "metrics_collected": { + "disk": { "measurement": ["used_percent"], "metrics_collection_interval": 60, "resources": ["/"] }, + "mem": { "measurement": ["mem_used_percent"], "metrics_collection_interval": 60 } + } + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { "file_path": "/home/ubuntu/searchengine/logs/app.log", "log_group_name": "/searchengine/app", "log_stream_name": "{instance_id}", "retention_in_days": 30 }, + { "file_path": "/home/ubuntu/searchengine/logs/error.log", "log_group_name": "/searchengine/error", "log_stream_name": "{instance_id}", "retention_in_days": 30 }, + { "file_path": "/home/ubuntu/searchengine/logs/requests.log", "log_group_name": "/searchengine/requests", "log_stream_name": "{instance_id}", "retention_in_days": 30 }, + { "file_path": "/home/ubuntu/searchengine/logs/gunicorn-access.log", "log_group_name": "/searchengine/gunicorn-access", "log_stream_name": "{instance_id}", "retention_in_days": 30 }, + { "file_path": "/home/ubuntu/searchengine/logs/gunicorn-error.log", "log_group_name": "/searchengine/gunicorn-error", "log_stream_name": "{instance_id}", "retention_in_days": 30 } + ] + } + } + } + } + EOF + cat << 'EOF' > tmp/nginx-proxy.conf + server { + server_name betasearch-api.tradhox.com search-api.tradhox.com; + + location / { + proxy_pass http://127.0.0.1:8087; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + listen 443 ssl; + ssl_certificate /etc/letsencrypt/live/betapartner-api.tradhox.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/betapartner-api.tradhox.com/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + } + + server { + listen 80; + server_name betasearch-api.tradhox.com search-api.tradhox.com; + return 301 https://$host$request_uri; + } + EOF + + - name: Package Application + run: | + tar -czf app.tar.gz --exclude=.git --exclude=tmp . + + - name: Deploy to Beta EC2 + run: | + apk add --no-cache openssh-client aws-cli + mkdir -p ~/.ssh + echo "${{ secrets.BETA_EC2_SSH_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Copy app archive and configuration files + scp -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa app.tar.gz tmp/amazon-cloudwatch-agent.json tmp/nginx-proxy.conf searchengine.service ${{ secrets.BETA_EC2_USERNAME }}@${{ secrets.BETA_EC2_HOST }}:/tmp/ + + # Execute deployment commands on remote server + ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa ${{ secrets.BETA_EC2_USERNAME }}@${{ secrets.BETA_EC2_HOST }} << 'EOF' + set -e + + # ── 1. Install Host Dependencies ────────────────────────────── + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv python3-dev pkg-config libpq-dev build-essential nginx + + # ── 2. Install Typesense Server Binary (Arch Detected) ───────── + if ! command -v typesense-server &> /dev/null; then + ARCH=$(dpkg --print-architecture) + echo "Installing Typesense Server v27.1 for architecture: ${ARCH}..." + curl -O https://dl.typesense.org/releases/27.1/typesense-server-27.1-${ARCH}.deb + sudo dpkg -i typesense-server-27.1-${ARCH}.deb + rm -f typesense-server-27.1-${ARCH}.deb + fi + + # Ensure Typesense directory & config exists + sudo mkdir -p /var/lib/typesense + cat << 'TYPESENSE_CONF' | sudo tee /etc/typesense/typesense-server.ini + api-key = vignesh_typesense_master_key_2026 + data-dir = /var/lib/typesense + log-dir = /var/log/typesense + api-port = 8108 + listen-address = 127.0.0.1 + TYPESENSE_CONF + + sudo systemctl enable typesense-server + sudo systemctl restart typesense-server || true + + # ── 3. Extract codebase to target directory ─────────────────── + sudo mkdir -p /home/ubuntu/searchengine + sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine + tar -xzf /tmp/app.tar.gz -C /home/ubuntu/searchengine + mkdir -p /home/ubuntu/searchengine/logs + sudo chown -R ubuntu:ubuntu /home/ubuntu/searchengine/logs + + # ── 4. Set up python virtual environment & dependencies ──────── + cd /home/ubuntu/searchengine + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + python manage.py migrate --noinput + + # ── 5. Systemd Service Deployment ────────────────────────────── + sudo cp /tmp/searchengine.service /etc/systemd/system/searchengine.service + sudo systemctl daemon-reload + sudo systemctl enable searchengine.service + sudo systemctl restart searchengine.service || { + echo "searchengine.service failed to restart! Checking journalctl:" + sudo journalctl -u searchengine.service -n 50 --no-pager + exit 1 + } + + # ── 6. Nginx Config ─────────────────────────────────────────── + if [ -f "/etc/letsencrypt/live/betasearch-api.tradhox.com/fullchain.pem" ]; then + sed -i 's|/etc/letsencrypt/live/betapartner-api.tradhox.com|/etc/letsencrypt/live/betasearch-api.tradhox.com|g' /tmp/nginx-proxy.conf + fi + sudo cp /tmp/nginx-proxy.conf /etc/nginx/sites-available/searchengine + sudo ln -sf /etc/nginx/sites-available/searchengine /etc/nginx/sites-enabled/searchengine + sudo nginx -t && sudo systemctl reload nginx + + # ── 7. CloudWatch Agent Config ──────────────────────────────── + sudo mkdir -p /opt/aws/amazon-cloudwatch-agent/etc/ + sudo cp /tmp/amazon-cloudwatch-agent.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json + sudo systemctl restart amazon-cloudwatch-agent || true + + echo "searchengine microservice successfully deployed!" + EOF diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab3e8ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,164 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..ab7a56b --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# searchengine + diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/apps.py b/api/apps.py new file mode 100644 index 0000000..6ad9776 --- /dev/null +++ b/api/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + +class ApiConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'api' diff --git a/api/management/__init__.py b/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/__init__.py b/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/management/commands/reindex_typesense.py b/api/management/commands/reindex_typesense.py new file mode 100644 index 0000000..210fcce --- /dev/null +++ b/api/management/commands/reindex_typesense.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand +from django.db import connection +from api.typesense_client import TypesenseService +import time + +class Command(BaseCommand): + help = "Reindex all products from PostgreSQL database into Typesense search index." + + def add_arguments(self, parser): + parser.add_argument( + '--batch-size', + type=int, + default=250, + help='Number of records to index per batch request.' + ) + + def handle(self, *args, **options): + batch_size = options.get('batch_size', 250) + self.stdout.write(self.style.SUCCESS("Ensuring Typesense schema...")) + TypesenseService.ensure_schema(force=True) + + count = 0 + batch = [] + now_ts = int(time.time()) + + self.stdout.write(self.style.SUCCESS("Fetching products from PostgreSQL (api_product)...")) + with connection.cursor() as cursor: + cursor.execute(""" + SELECT id, sku, title, description, category, supplier_id, price, + compare_at_price, stock, image, is_gi_tagged, is_active, status + FROM api_product + """) + columns = [col[0] for col in cursor.description] + rows = cursor.fetchall() + total_rows = len(rows) + self.stdout.write(f"Found {total_rows} products to index.") + + for idx, row in enumerate(rows): + item = dict(zip(columns, row)) + item['created_at'] = now_ts + idx + batch.append(item) + + if len(batch) >= batch_size: + TypesenseService.index_products_batch(batch) + count += len(batch) + self.stdout.write(f"Indexed batch: {count}/{total_rows} products...") + batch = [] + + if batch: + TypesenseService.index_products_batch(batch) + count += len(batch) + self.stdout.write(f"Indexed final batch: {count}/{total_rows} products...") + + self.stdout.write(self.style.SUCCESS(f"Successfully reindexed {count} products into Typesense.")) diff --git a/api/migrations/0001_initial.py b/api/migrations/0001_initial.py new file mode 100644 index 0000000..4906a2e --- /dev/null +++ b/api/migrations/0001_initial.py @@ -0,0 +1,24 @@ +# Generated by Django 6.1 on 2026-10-15 14:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ProductIndexLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sku', models.CharField(max_length=100, unique=True)), + ('status', models.CharField(default='synced', max_length=50)), + ('last_synced_at', models.DateTimeField(auto_now=True)), + ('error', models.TextField(blank=True, null=True)), + ], + ), + ] diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/models.py b/api/models.py new file mode 100644 index 0000000..1a63fd0 --- /dev/null +++ b/api/models.py @@ -0,0 +1,11 @@ +from django.db import models + +# SearchEngine microservice references products directly from PostgreSQL productprofile db or typesense index +class ProductIndexLog(models.Model): + sku = models.CharField(max_length=100, unique=True) + status = models.CharField(max_length=50, default='synced') + last_synced_at = models.DateTimeField(auto_now=True) + error = models.TextField(blank=True, null=True) + + def __str__(self): + return f"{self.sku} - {self.status}" diff --git a/api/tests.py b/api/tests.py new file mode 100644 index 0000000..534ea14 --- /dev/null +++ b/api/tests.py @@ -0,0 +1,134 @@ +from django.test import TestCase +from rest_framework.test import APIClient +from unittest.mock import patch, MagicMock +from api.typesense_client import ( + TypesenseService, + safe_str, + safe_int, + safe_float, + safe_bool, +) + +class TypesenseServiceUnitTests(TestCase): + def test_safe_helpers(self): + self.assertEqual(safe_str(None, "default"), "default") + self.assertEqual(safe_str(123), "123") + self.assertEqual(safe_int(None, 10), 10) + self.assertEqual(safe_int("42"), 42) + self.assertEqual(safe_int("invalid", 5), 5) + self.assertEqual(safe_float(None, 3.14), 3.14) + self.assertEqual(safe_float("99.95"), 99.95) + self.assertEqual(safe_float("invalid", 0.0), 0.0) + self.assertTrue(safe_bool("true")) + self.assertTrue(safe_bool("1")) + self.assertFalse(safe_bool("false")) + self.assertFalse(safe_bool("0")) + self.assertFalse(safe_bool(None, False)) + + def test_prepare_product_doc_handles_nulls_gracefully(self): + raw_product = { + 'id': 101, + 'sku': 'TRDX-TEST-001', + 'title': 'Test Silk Saree', + 'description': None, + 'category': None, + 'supplier_id': None, + 'supplier_name': None, + 'price': None, + 'compare_at_price': None, + 'stock': None, + 'image': None, + 'is_gi_tagged': None, + 'is_active': None, + 'status': None, + 'created_at': None + } + doc = TypesenseService.prepare_product_doc(raw_product) + self.assertEqual(doc['id'], '101') + self.assertEqual(doc['sku'], 'TRDX-TEST-001') + self.assertEqual(doc['title'], 'Test Silk Saree') + self.assertEqual(doc['description'], '') + self.assertEqual(doc['category'], 'Uncategorized') + self.assertEqual(doc['supplier_id'], 0) + self.assertEqual(doc['price'], 0.0) + self.assertEqual(doc['compare_at_price'], 0.0) + self.assertEqual(doc['stock'], 0) + self.assertFalse(doc['is_gi_tagged']) + self.assertTrue(doc['is_active']) + self.assertEqual(doc['status'], 'active') + self.assertEqual(doc['created_at'], 0) + + +class SearchAndIndexApiTests(TestCase): + def setUp(self): + self.client = APIClient() + + @patch.object(TypesenseService, 'search_products') + def test_search_view_defaults(self, mock_search): + mock_search.return_value = { + 'hits': [{'document': {'id': '1', 'title': 'Product 1'}}], + 'found': 1, + 'facet_counts': [] + } + + response = self.client.get('/api/search/', {'q': 'saree'}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 1) + self.assertEqual(len(response.data['results']), 1) + mock_search.assert_called_once() + _, kwargs = mock_search.call_args + self.assertEqual(kwargs['query'], 'saree') + self.assertIn('is_active:=true', kwargs['filter_by']) + self.assertEqual(kwargs['sort_by'], '_text_match:desc,created_at:desc') + + @patch.object(TypesenseService, 'search_products') + def test_search_view_filters_and_sorting(self, mock_search): + mock_search.return_value = {'hits': [], 'found': 0, 'facet_counts': []} + + response = self.client.get('/api/search/', { + 'q': '*', + 'category': 'Textiles & Silk', + 'min_price': '100', + 'max_price': '500', + 'is_gi_tagged': 'false', + 'supplier_id': '42', + 'sort_by': 'price_asc', + 'page': '2', + 'per_page': '10' + }) + self.assertEqual(response.status_code, 200) + _, kwargs = mock_search.call_args + self.assertIn('category:="Textiles & Silk"', kwargs['filter_by']) + self.assertIn('price:[100.0..500.0]', kwargs['filter_by']) + self.assertIn('is_gi_tagged:=false', kwargs['filter_by']) + self.assertIn('supplier_id:=42', kwargs['filter_by']) + self.assertEqual(kwargs['sort_by'], 'price:asc') + self.assertEqual(kwargs['page'], 2) + self.assertEqual(kwargs['per_page'], 10) + + @patch.object(TypesenseService, 'index_product') + def test_index_single_product(self, mock_index): + mock_index.return_value = {'success': True} + payload = {'id': 1, 'sku': 'SKU123', 'title': 'Test Item', 'price': 99.0} + response = self.client.post('/api/index/product/', payload, format='json') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['status'], 'indexed') + + @patch.object(TypesenseService, 'index_products_batch') + def test_index_batch_products(self, mock_batch): + mock_batch.return_value = [{'success': True}, {'success': True}] + payload = [ + {'id': 1, 'sku': 'SKU1', 'title': 'Item 1'}, + {'id': 2, 'sku': 'SKU2', 'title': 'Item 2'} + ] + response = self.client.post('/api/index/product/', payload, format='json') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 2) + + @patch.object(TypesenseService, 'delete_product') + def test_delete_product(self, mock_delete): + mock_delete.return_value = {'deleted': True} + response = self.client.delete('/api/index/product/TRDX-001/') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['status'], 'deleted') + mock_delete.assert_called_once_with('TRDX-001') diff --git a/api/typesense_client.py b/api/typesense_client.py new file mode 100644 index 0000000..bccce83 --- /dev/null +++ b/api/typesense_client.py @@ -0,0 +1,176 @@ +import typesense +from django.conf import settings +import logging + +logger = logging.getLogger(__name__) + +PRODUCTS_SCHEMA = { + 'name': 'products', + 'fields': [ + {'name': 'id', 'type': 'string'}, + {'name': 'title', 'type': 'string'}, + {'name': 'description', 'type': 'string', 'optional': True}, + {'name': 'category', 'type': 'string', 'facet': True}, + {'name': 'supplier_id', 'type': 'int32', 'facet': True}, + {'name': 'supplier_name', 'type': 'string', 'optional': True}, + {'name': 'price', 'type': 'float', 'facet': True}, + {'name': 'compare_at_price', 'type': 'float', 'optional': True}, + {'name': 'stock', 'type': 'int32'}, + {'name': 'sku', 'type': 'string'}, + {'name': 'image', 'type': 'string', 'optional': True}, + {'name': 'is_gi_tagged', 'type': 'bool', 'facet': True}, + {'name': 'is_active', 'type': 'bool', 'facet': True}, + {'name': 'status', 'type': 'string', 'facet': True}, + {'name': 'created_at', 'type': 'int64', 'optional': True}, + ], + 'default_sorting_field': 'created_at' +} + +def safe_str(val, default=""): + if val is None: + return default + return str(val) + +def safe_int(val, default=0): + if val is None or val == "": + return default + try: + return int(val) + except (ValueError, TypeError): + return default + +def safe_float(val, default=0.0): + if val is None or val == "": + return default + try: + return float(val) + except (ValueError, TypeError): + return default + +def safe_bool(val, default=False): + if val is None: + return default + if isinstance(val, bool): + return val + if isinstance(val, (int, float)): + return bool(val) + if isinstance(val, str): + return val.lower() in ("true", "1", "yes") + return default + + +class TypesenseService: + _client = None + _schema_initialized = False + + @classmethod + def get_client(cls): + if cls._client is None: + cfg = getattr(settings, 'TYPESENSE_CONFIG', {}) + cls._client = typesense.Client({ + 'nodes': [{ + 'host': cfg.get('HOST', '127.0.0.1'), + 'port': str(cfg.get('PORT', '8108')), + 'protocol': cfg.get('PROTOCOL', 'http') + }], + 'api_key': cfg.get('API_KEY', 'vignesh_typesense_master_key_2026'), + 'connection_timeout_seconds': cfg.get('TIMEOUT', 5.0) + }) + return cls._client + + @classmethod + def ensure_schema(cls, force=False): + if cls._schema_initialized and not force: + return + client = cls.get_client() + try: + client.collections['products'].retrieve() + logger.info("Typesense 'products' collection already exists.") + cls._schema_initialized = True + except Exception: + logger.info("Creating Typesense 'products' collection schema...") + client.collections.create(PRODUCTS_SCHEMA) + cls._schema_initialized = True + + @classmethod + def prepare_product_doc(cls, product_dict): + doc_id = product_dict.get('id') + if doc_id is None: + doc_id = product_dict.get('sku') + + return { + 'id': safe_str(doc_id), + 'title': safe_str(product_dict.get('title')), + 'description': safe_str(product_dict.get('description')), + 'category': safe_str(product_dict.get('category'), 'Uncategorized'), + 'supplier_id': safe_int(product_dict.get('supplier_id'), 0), + 'supplier_name': safe_str(product_dict.get('supplier_name')), + 'price': safe_float(product_dict.get('price'), 0.0), + 'compare_at_price': safe_float(product_dict.get('compare_at_price'), 0.0), + 'stock': safe_int(product_dict.get('stock'), 0), + 'sku': safe_str(product_dict.get('sku')), + 'image': safe_str(product_dict.get('image')), + 'is_gi_tagged': safe_bool(product_dict.get('is_gi_tagged'), False), + 'is_active': safe_bool(product_dict.get('is_active'), True), + 'status': safe_str(product_dict.get('status'), 'active'), + 'created_at': safe_int(product_dict.get('created_at'), 0) + } + + @classmethod + def index_product(cls, product_dict): + cls.ensure_schema() + client = cls.get_client() + doc = cls.prepare_product_doc(product_dict) + return client.collections['products'].documents.upsert(doc) + + @classmethod + def index_products_batch(cls, products_list): + if not products_list: + return [] + cls.ensure_schema() + client = cls.get_client() + docs = [cls.prepare_product_doc(p) for p in products_list] + return client.collections['products'].documents.import_(docs, {'action': 'upsert'}) + + @classmethod + def delete_product(cls, product_id): + client = cls.get_client() + pid_str = str(product_id).strip() + try: + return client.collections['products'].documents[pid_str].delete() + except Exception as e: + logger.info(f"Direct delete by ID {pid_str} failed ({e}), attempting delete by SKU filter...") + try: + return client.collections['products'].documents.delete({'filter_by': f'sku:="{pid_str}"'}) + except Exception as e_filter: + logger.warning(f"Could not delete document with identifier {pid_str} from Typesense: {e_filter}") + return None + + @classmethod + def search_products(cls, query="*", filter_by="", sort_by="", page=1, per_page=20): + cls.ensure_schema() + client = cls.get_client() + q_clean = query.strip() if query else "*" + + search_parameters = { + 'q': q_clean if q_clean else '*', + 'query_by': 'title,description,category,sku', + 'facet_by': 'category,is_gi_tagged,supplier_id', + 'page': page, + 'per_page': per_page, + 'num_typos': 2, + 'prefix': True + } + + if filter_by: + search_parameters['filter_by'] = filter_by + + if sort_by: + search_parameters['sort_by'] = sort_by + else: + if q_clean and q_clean != '*': + search_parameters['sort_by'] = '_text_match:desc,created_at:desc' + else: + search_parameters['sort_by'] = 'created_at:desc' + + return client.collections['products'].documents.search(search_parameters) diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 0000000..a945902 --- /dev/null +++ b/api/urls.py @@ -0,0 +1,10 @@ +from django.urls import path +from .views import ProductSearchView, IndexProductView, DeleteProductView, ReindexCatalogView + +urlpatterns = [ + path('search/', ProductSearchView.as_view(), name='product-search'), + path('products/search/', ProductSearchView.as_view(), name='product-search-alias'), + path('index/product/', IndexProductView.as_view(), name='index-product'), + path('index/product//', DeleteProductView.as_view(), name='delete-product'), + path('index/reindex/', ReindexCatalogView.as_view(), name='reindex-catalog'), +] diff --git a/api/views.py b/api/views.py new file mode 100644 index 0000000..8a1c394 --- /dev/null +++ b/api/views.py @@ -0,0 +1,175 @@ +from rest_framework.views import APIView +from rest_framework.response import Response +from rest_framework import status +from django.db import connection +from .typesense_client import TypesenseService +import logging +import time + +logger = logging.getLogger(__name__) + +class ProductSearchView(APIView): + def get(self, request): + try: + q = request.query_params.get('q', '*').strip() + category = request.query_params.get('category', '').strip() + min_price = request.query_params.get('min_price') + max_price = request.query_params.get('max_price') + is_gi_tagged = request.query_params.get('is_gi_tagged') + supplier_id = request.query_params.get('supplier_id') + sort_by_param = request.query_params.get('sort_by', '').strip() + + try: + page = max(1, int(request.query_params.get('page', 1))) + per_page = min(250, max(1, int(request.query_params.get('per_page', 20)))) + except (ValueError, TypeError): + return Response({'error': 'Invalid page or per_page parameter'}, status=status.HTTP_400_BAD_REQUEST) + + filter_clauses = ['is_active:=true'] + + if category: + escaped_category = category.replace('"', '\\"') + filter_clauses.append(f'category:="{escaped_category}"') + + if supplier_id: + try: + s_id = int(supplier_id) + filter_clauses.append(f'supplier_id:={s_id}') + except (ValueError, TypeError): + return Response({'error': 'supplier_id must be an integer'}, status=status.HTTP_400_BAD_REQUEST) + + if is_gi_tagged is not None: + val = is_gi_tagged.strip().lower() + if val in ('true', '1'): + filter_clauses.append('is_gi_tagged:=true') + elif val in ('false', '0'): + filter_clauses.append('is_gi_tagged:=false') + + if min_price is not None or max_price is not None: + try: + p_min = float(min_price) if min_price else 0.0 + p_max = float(max_price) if max_price else 9999999.0 + if p_min > p_max: + p_min, p_max = p_max, p_min + filter_clauses.append(f'price:[{p_min}..{p_max}]') + except (ValueError, TypeError): + return Response({'error': 'Price filters must be valid numbers'}, status=status.HTTP_400_BAD_REQUEST) + + filter_by_str = ' && '.join(filter_clauses) + + if sort_by_param == 'price_asc': + sort_by_str = 'price:asc' + elif sort_by_param == 'price_desc': + sort_by_str = 'price:desc' + elif sort_by_param == 'relevance': + sort_by_str = '_text_match:desc,created_at:desc' + elif sort_by_param == 'newest': + sort_by_str = 'created_at:desc' + else: + sort_by_str = '_text_match:desc,created_at:desc' if (q and q != '*') else 'created_at:desc' + + res = TypesenseService.search_products( + query=q, + filter_by=filter_by_str, + sort_by=sort_by_str, + page=page, + per_page=per_page + ) + + hits = res.get('hits', []) + found = res.get('found', 0) + total_pages = (found + per_page - 1) // per_page if per_page > 0 else 1 + + results = [hit.get('document', {}) for hit in hits] + facet_counts = res.get('facet_counts', []) + + return Response({ + 'count': found, + 'page': page, + 'per_page': per_page, + 'total_pages': total_pages, + 'facets': facet_counts, + 'results': results + }, status=status.HTTP_200_OK) + except Exception as e: + logger.error(f"Typesense Search Error: {e}", exc_info=True) + return Response({'error': str(e), 'results': [], 'count': 0}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class IndexProductView(APIView): + def post(self, request): + try: + data = request.data + if not data: + return Response({'error': 'Payload cannot be empty'}, status=status.HTTP_400_BAD_REQUEST) + + if isinstance(data, list): + valid_items = [] + for item in data: + if isinstance(item, dict) and ('id' in item or 'sku' in item): + valid_items.append(item) + if not valid_items: + return Response({'error': 'Each product in list must have an id or sku'}, status=status.HTTP_400_BAD_REQUEST) + res = TypesenseService.index_products_batch(valid_items) + return Response({'status': 'indexed', 'count': len(valid_items), 'result': res}, status=status.HTTP_200_OK) + elif isinstance(data, dict): + if 'id' not in data and 'sku' not in data: + return Response({'error': 'Product id or sku is required'}, status=status.HTTP_400_BAD_REQUEST) + res = TypesenseService.index_product(data) + return Response({'status': 'indexed', 'result': res}, status=status.HTTP_200_OK) + else: + return Response({'error': 'Invalid JSON format. Expected JSON object or array.'}, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + logger.error(f"Product Index Error: {e}", exc_info=True) + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class DeleteProductView(APIView): + def delete(self, request, product_id): + try: + if not product_id or not str(product_id).strip(): + return Response({'error': 'Product identifier is required'}, status=status.HTTP_400_BAD_REQUEST) + res = TypesenseService.delete_product(product_id) + return Response({'status': 'deleted', 'result': res}, status=status.HTTP_200_OK) + except Exception as e: + logger.error(f"Product Delete Error: {e}", exc_info=True) + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class ReindexCatalogView(APIView): + def post(self, request): + try: + TypesenseService.ensure_schema(force=True) + indexed_count = 0 + batch_size = 250 + batch = [] + + with connection.cursor() as cursor: + cursor.execute(""" + SELECT id, sku, title, description, category, supplier_id, price, + compare_at_price, stock, image, is_gi_tagged, is_active, status + FROM api_product + """) + columns = [col[0] for col in cursor.description] + rows = cursor.fetchall() + now_ts = int(time.time()) + + for idx, row in enumerate(rows): + item = dict(zip(columns, row)) + # Retain descending ordering with timestamp offset + item['created_at'] = now_ts + idx + batch.append(item) + + if len(batch) >= batch_size: + TypesenseService.index_products_batch(batch) + indexed_count += len(batch) + batch = [] + + if batch: + TypesenseService.index_products_batch(batch) + indexed_count += len(batch) + + return Response({'status': 'completed', 'total_indexed': indexed_count}, status=status.HTTP_200_OK) + except Exception as e: + logger.error(f"Full Reindex Error: {e}", exc_info=True) + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/asgi.py b/config/asgi.py new file mode 100644 index 0000000..38396a8 --- /dev/null +++ b/config/asgi.py @@ -0,0 +1,4 @@ +import os +from django.core.asgi import get_asgi_application +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') +application = get_asgi_application() diff --git a/config/settings.py b/config/settings.py new file mode 100644 index 0000000..7e3f5be --- /dev/null +++ b/config/settings.py @@ -0,0 +1,90 @@ +# Django settings for searchengine microservice +import os +from pathlib import Path + +BASE_DIR = Path(__file__).resolve().parent.parent + +SECRET_KEY = os.environ.get('SECRET_KEY', 'django-insecure-tradhox-searchengine-key-2026') +DEBUG = os.environ.get('DEBUG', 'True').lower() in ('true', '1') +ALLOWED_HOSTS = ['*'] + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'corsheaders', + 'rest_framework', + 'api', +] + +MIDDLEWARE = [ + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'config.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'config.wsgi.application' + +# Database Configuration (PostgreSQL Exclusive) +db_host = os.environ.get('DB_HOST', '16.113.106.152') +db_port = int(os.environ.get('DB_PORT', '5432')) +db_user = os.environ.get('DB_USER', 'vignesh') +db_pass = os.environ.get('DB_PASSWORD', 'vignesh_secret_pass_2026') +env_name = os.environ.get('ENVIRONMENT', 'beta') +db_suffix = f'_{env_name}' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': f'productprofile{db_suffix}', + 'USER': db_user, + 'PASSWORD': db_pass, + 'HOST': db_host, + 'PORT': str(db_port), + } +} + +AUTH_PASSWORD_VALIDATORS = [] + +LANGUAGE_CODE = 'en-us' +TIME_ZONE = 'UTC' +USE_I18N = True +USE_TZ = True + +STATIC_URL = 'static/' +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +CORS_ALLOW_ALL_ORIGINS = True + +TYPESENSE_CONFIG = { + 'HOST': os.environ.get('TYPESENSE_HOST', '127.0.0.1'), + 'PORT': os.environ.get('TYPESENSE_PORT', '8108'), + 'PROTOCOL': os.environ.get('TYPESENSE_PROTOCOL', 'http'), + 'API_KEY': os.environ.get('TYPESENSE_API_KEY', 'vignesh_typesense_master_key_2026'), + 'TIMEOUT': 5.0 +} diff --git a/config/urls.py b/config/urls.py new file mode 100644 index 0000000..3cbb08b --- /dev/null +++ b/config/urls.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from django.urls import path, include + +urlpatterns = [ + path('admin/', admin.site.urls), + path('api/', include('api.urls')), +] diff --git a/config/wsgi.py b/config/wsgi.py new file mode 100644 index 0000000..5eebe82 --- /dev/null +++ b/config/wsgi.py @@ -0,0 +1,4 @@ +import os +from django.core.wsgi import get_wsgi_application +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') +application = get_wsgi_application() diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..6f6eb9d --- /dev/null +++ b/logs/.gitkeep @@ -0,0 +1 @@ +# Keep logs directory tracked in version control diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..23ba547 --- /dev/null +++ b/manage.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import os +import sys + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9fed405 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Django==6.1 +djangorestframework +django-cors-headers +gunicorn==26.0.0 +psycopg2-binary +requests +typesense>=0.21.0 diff --git a/searchengine.service b/searchengine.service new file mode 100644 index 0000000..0bf87a5 --- /dev/null +++ b/searchengine.service @@ -0,0 +1,16 @@ +[Unit] +Description=Tradhox Search Engine Microservice Daemon +After=network.target typesense-server.service + +[Service] +User=ubuntu +Group=ubuntu +WorkingDirectory=/home/ubuntu/searchengine +ExecStart=/home/ubuntu/searchengine/venv/bin/gunicorn --workers 3 --bind 0.0.0.0:8087 config.wsgi:application +Restart=always +RestartSec=5 +StandardOutput=append:/home/ubuntu/searchengine/logs/gunicorn-access.log +StandardError=append:/home/ubuntu/searchengine/logs/gunicorn-error.log + +[Install] +WantedBy=multi-user.target