From 14cd99fb3c3622f95b671dd4739f0993b1681050 Mon Sep 17 00:00:00 2001 From: Ashwin Date: Fri, 2 Oct 2026 12:23:22 +0530 Subject: [PATCH] feat: Implement per-client and global rate limits for magic-link and portal sign-in requests - Added `FirstRefusalThrottleMixin` to various views to prevent a single client from consuming shared throttle limits. - Introduced new throttles: `MagicLinkIPThrottle`, `MagicLinkGlobalThrottle`, `PortalLoginIPThrottle`, and `PortalLoginOrgThrottle`. - Updated environment variables in `.env.example` to include new throttle settings. - Enhanced tests to verify that refused clients do not affect global limits. - Updated documentation to reflect changes in authentication and throttling behavior. --- backend/.env.example | 13 +- backend/cases/help_center_views.py | 3 +- .../cases/tests/test_help_center_public.py | 23 +++ backend/common/tests/test_calendar_feed.py | 23 +++ backend/common/tests/test_magic_link.py | 10 + backend/common/tests/test_portal_login.py | 10 + .../common/tests/test_sign_in_throttles.py | 181 ++++++++++++++++++ backend/common/throttles.py | 78 ++++++++ backend/common/views/auth_views.py | 11 +- backend/common/views/calendar_feed_views.py | 3 +- backend/common/views/portal_auth_views.py | 14 +- backend/crm/settings.py | 24 ++- backend/pyproject.toml | 2 +- backend/uv.lock | 2 +- backend/webforms/public_views.py | 3 +- backend/webforms/tests/test_submit.py | 32 ++++ backend/webforms/throttles.py | 5 +- docs/api/authentication.md | 5 + docs/api/customer-portal.md | 5 +- docs/api/errors.md | 12 +- docs/reference/environment-variables.md | 8 +- 21 files changed, 441 insertions(+), 26 deletions(-) create mode 100644 backend/common/tests/test_sign_in_throttles.py create mode 100644 backend/common/throttles.py diff --git a/backend/.env.example b/backend/.env.example index f0df5ed2..006c1889 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -65,8 +65,8 @@ TRUST_PROXY_SSL_HEADER=False NUM_PROXIES= # Shared secret with the SvelteKit servers that relay anonymous visitors (the -# web app's help center and estimate portal, the marketing site's contact -# form). A request carrying it in X-BottleCRM-Relay-Secret is believed about +# web app's sign-in pages, help center and estimate portal, the marketing +# site's contact form). A request carrying it in X-BottleCRM-Relay-Secret is believed about # the visitor address in X-BottleCRM-Client-IP, so each visitor gets their own # per-IP throttle bucket and recorded address instead of the relay's. Set the # same value as RELAY_SECRET on each SvelteKit service. At least 32 characters @@ -84,3 +84,12 @@ CACHE_URL="" # X-Forwarded-For rotation cannot evade. WEBFORM_THROTTLE_IP="10/hour" WEBFORM_THROTTLE_GLOBAL="200/day" + +# Magic-link and customer portal sign-in request limits. The first of each is +# per client IP; without RELAY_SECRET every web sign-in shares the SvelteKit +# server's bucket, so keep it roomy. The global one pauses sign-in email for +# everyone when reached; the portal one, for one org. +MAGIC_LINK_THROTTLE_IP="30/hour" +MAGIC_LINK_THROTTLE_GLOBAL="300/hour" +PORTAL_LOGIN_THROTTLE_IP="30/hour" +PORTAL_LOGIN_THROTTLE_ORG="100/hour" diff --git a/backend/cases/help_center_views.py b/backend/cases/help_center_views.py index 9205c421..56998e8d 100644 --- a/backend/cases/help_center_views.py +++ b/backend/cases/help_center_views.py @@ -36,6 +36,7 @@ from cases.portal_views import published_articles, related_articles from common.models import Org from common.request_meta import client_ip from common.tasks import set_rls_context +from common.throttles import FirstRefusalThrottleMixin # A search term longer than this is not a question anybody typed. SEARCH_MAX = 200 @@ -91,7 +92,7 @@ class HelpCenterPagination(LimitOffsetPagination): max_limit = 100 -class PublicHelpCenterView(APIView): +class PublicHelpCenterView(FirstRefusalThrottleMixin, APIView): authentication_classes: list = [] permission_classes = (AllowAny,) throttle_classes = [HelpCenterIPThrottle, HelpCenterGlobalThrottle] diff --git a/backend/cases/tests/test_help_center_public.py b/backend/cases/tests/test_help_center_public.py index c1ca759b..0fda9a9f 100644 --- a/backend/cases/tests/test_help_center_public.py +++ b/backend/cases/tests/test_help_center_public.py @@ -313,6 +313,29 @@ class TestThrottle: ] assert codes == [200, 200, 429] + def test_a_refused_client_does_not_spend_the_global_budget( + self, anon, acme, monkeypatch + ): + """DRF consults every throttle even after one refuses, so without + FirstRefusalThrottleMixin one address could fill a help center's + shared bucket and lock every reader out.""" + monkeypatch.setattr( + HelpCenterIPThrottle, "THROTTLE_RATES", {"help_center_ip": "1/hour"} + ) + monkeypatch.setattr( + HelpCenterGlobalThrottle, + "THROTTLE_RATES", + {"help_center_global": "3/hour"}, + ) + flood = [ + anon.get(list_url(), REMOTE_ADDR="198.51.100.1").status_code + for _ in range(5) + ] + assert flood == [200, 429, 429, 429, 429] + assert anon.get(list_url(), REMOTE_ADDR="198.51.100.2").status_code == 200 + assert anon.get(list_url(), REMOTE_ADDR="198.51.100.3").status_code == 200 + assert anon.get(list_url(), REMOTE_ADDR="198.51.100.4").status_code == 429 + def test_global_limit_is_per_help_center(self, anon, acme, other, monkeypatch): """One org's traffic never locks readers out of another org's pages.""" monkeypatch.setattr( diff --git a/backend/common/tests/test_calendar_feed.py b/backend/common/tests/test_calendar_feed.py index a36fcb6d..7ee78ecf 100644 --- a/backend/common/tests/test_calendar_feed.py +++ b/backend/common/tests/test_calendar_feed.py @@ -484,6 +484,29 @@ class TestThrottle: ] assert PublicCalendarFeedView.authentication_classes == [] + def test_a_refused_address_does_not_spend_the_feeds_budget( + self, user_client, anon, monkeypatch + ): + """DRF consults every throttle even after one refuses, so without + FirstRefusalThrottleMixin one address hammering a feed used up that + feed's allowance for every other calendar fetching it.""" + monkeypatch.setattr( + CalendarFeedIPThrottle, "THROTTLE_RATES", {"calendar_feed_ip": "1/hour"} + ) + monkeypatch.setattr( + CalendarFeedTokenThrottle, + "THROTTLE_RATES", + {"calendar_feed_token": "3/hour"}, + ) + url = _path(_enable(user_client)) + flood = [ + anon.get(url, REMOTE_ADDR="198.51.100.1").status_code for _ in range(5) + ] + assert flood == [200, 429, 429, 429, 429] + assert anon.get(url, REMOTE_ADDR="198.51.100.2").status_code == 200 + assert anon.get(url, REMOTE_ADDR="198.51.100.3").status_code == 200 + assert anon.get(url, REMOTE_ADDR="198.51.100.4").status_code == 429 + def test_per_token_limit(self, user_client, admin_client, anon, monkeypatch): monkeypatch.setattr( CalendarFeedTokenThrottle, diff --git a/backend/common/tests/test_magic_link.py b/backend/common/tests/test_magic_link.py index f92e7443..91670bee 100644 --- a/backend/common/tests/test_magic_link.py +++ b/backend/common/tests/test_magic_link.py @@ -9,12 +9,22 @@ from datetime import timedelta from unittest.mock import patch import pytest +from django.core.cache import cache from django.utils import timezone from rest_framework import status from common.models import MagicLinkToken, Profile, User +@pytest.fixture(autouse=True) +def _fresh_throttle(): + """The request endpoints are throttled per client, and every test here + calls from the same address.""" + cache.clear() + yield + cache.clear() + + @pytest.mark.django_db class TestMagicLinkRequest: """Tests for POST /api/auth/magic-link/request/""" diff --git a/backend/common/tests/test_portal_login.py b/backend/common/tests/test_portal_login.py index 3bfc47ff..8133e892 100644 --- a/backend/common/tests/test_portal_login.py +++ b/backend/common/tests/test_portal_login.py @@ -4,6 +4,7 @@ from datetime import timedelta import pytest from django.contrib.auth.hashers import check_password, make_password +from django.core.cache import cache from django.utils import timezone from rest_framework.test import APIClient @@ -11,6 +12,15 @@ from common.models import PortalLoginToken from contacts.models import Contact +@pytest.fixture(autouse=True) +def _fresh_throttle(): + """The request endpoints are throttled per client, and every test here + calls from the same address.""" + cache.clear() + yield + cache.clear() + + @pytest.fixture def contact(org_a): return Contact.objects.create( diff --git a/backend/common/tests/test_sign_in_throttles.py b/backend/common/tests/test_sign_in_throttles.py new file mode 100644 index 00000000..705b6f93 --- /dev/null +++ b/backend/common/tests/test_sign_in_throttles.py @@ -0,0 +1,181 @@ +"""Per-client and shared rate limits on the anonymous sign-in request endpoints. + +The per-recipient cap (five an hour per email or contact) is tested beside each +view. These cover the layers in `common.throttles` that stop a caller from +getting a fresh bucket by changing the address they ask for. +""" + +from unittest.mock import patch + +import pytest +from django.core.cache import cache +from django.test import override_settings +from rest_framework.test import APIClient + +from common.models import MagicLinkToken, PortalLoginToken +from common.request_meta import RELAY_CLIENT_IP_HEADER, RELAY_SECRET_HEADER +from common.throttles import ( + FirstRefusalThrottleMixin, + MagicLinkGlobalThrottle, + MagicLinkIPThrottle, + PortalLoginIPThrottle, + PortalLoginOrgThrottle, +) +from common.views.auth_views import MagicLinkRequestView +from common.views.portal_auth_views import PortalLoginRequestView +from contacts.models import Contact + +MAGIC = "/api/auth/magic-link/request/" +SECRET = "s" * 40 + + +@pytest.fixture(autouse=True) +def _fresh_throttle(): + cache.clear() + yield + cache.clear() + + +def _rate(monkeypatch, throttle, rate): + monkeypatch.setattr(throttle, "THROTTLE_RATES", {throttle.scope: rate}) + + +def _magic(n, ip): + with patch("common.tasks.send_magic_link_email.delay") as send: + response = APIClient().post( + MAGIC, {"email": f"poc-{n}@example.com"}, format="json", REMOTE_ADDR=ip + ) + return response.status_code, send.call_count + + +def _portal(org, email, ip): + with patch("common.tasks.send_portal_login_email.delay") as send: + response = APIClient().post( + f"/api/portal/login/{org.id}/request/", + {"email": email}, + format="json", + REMOTE_ADDR=ip, + ) + return response.status_code, send.call_count + + +class TestWiring: + def test_views_are_throttled_per_client_first(self): + assert MagicLinkRequestView.throttle_classes == [ + MagicLinkIPThrottle, + MagicLinkGlobalThrottle, + ] + assert PortalLoginRequestView.throttle_classes == [ + PortalLoginIPThrottle, + PortalLoginOrgThrottle, + ] + assert issubclass(MagicLinkRequestView, FirstRefusalThrottleMixin) + assert issubclass(PortalLoginRequestView, FirstRefusalThrottleMixin) + + +class TestMagicLink: + def test_new_addresses_do_not_buy_a_new_bucket(self, monkeypatch): + """The report's PoC: one client, a different email each time.""" + _rate(monkeypatch, MagicLinkIPThrottle, "2/hour") + results = [_magic(n, "198.51.100.1") for n in range(3)] + assert results == [(200, 1), (200, 1), (429, 0)] + assert MagicLinkToken.objects.count() == 2 + # Another client is unaffected. + assert _magic(9, "198.51.100.2") == (200, 1) + + def test_global_cap_holds_across_clients(self, monkeypatch): + _rate(monkeypatch, MagicLinkGlobalThrottle, "2/hour") + results = [_magic(n, f"198.51.100.{n + 1}") for n in range(3)] + assert results == [(200, 1), (200, 1), (429, 0)] + + def test_a_refused_client_does_not_spend_the_global_budget(self, monkeypatch): + """Without FirstRefusalThrottleMixin DRF records every request the + per-IP limit refused in the global bucket too, and one address could + lock out everyone.""" + _rate(monkeypatch, MagicLinkIPThrottle, "1/hour") + _rate(monkeypatch, MagicLinkGlobalThrottle, "3/hour") + flood = [_magic(n, "198.51.100.1")[0] for n in range(5)] + assert flood == [200, 429, 429, 429, 429] + assert _magic(10, "198.51.100.2") == (200, 1) + assert _magic(11, "198.51.100.3") == (200, 1) + # The global cap itself still binds. + assert _magic(12, "198.51.100.4") == (429, 0) + + @override_settings(RELAY_SECRET=SECRET) + def test_relayed_visitors_get_their_own_buckets(self, monkeypatch): + """The web sign-in page calls from the SvelteKit server's address, so a + signed visitor address is what keeps all web sign-ins from sharing one + bucket. An unsigned one is ignored.""" + _rate(monkeypatch, MagicLinkIPThrottle, "1/hour") + + def relayed(n, visitor, secret=SECRET): + with patch("common.tasks.send_magic_link_email.delay"): + return ( + APIClient() + .post( + MAGIC, + {"email": f"web-{n}@example.com"}, + format="json", + REMOTE_ADDR="10.0.0.5", + **{ + RELAY_SECRET_HEADER: secret, + RELAY_CLIENT_IP_HEADER: visitor, + }, + ) + .status_code + ) + + assert relayed(1, "203.0.113.1") == 200 + assert relayed(2, "203.0.113.2") == 200 + assert relayed(3, "203.0.113.1") == 429 + # A forged visitor header without the secret lands in the relay's own + # bucket, which is still fresh, and then shares it. + assert relayed(4, "203.0.113.9", secret="wrong" * 10) == 200 + assert relayed(5, "203.0.113.10", secret="wrong" * 10) == 429 + + +class TestPortalLogin: + @pytest.fixture + def contacts(self, org_a, org_b): + return [ + Contact.objects.create( + org=org, + first_name="Pat", + last_name="Smith", + email=f"pat{n}@example.com", + ) + for org in (org_a, org_b) + for n in range(4) + ] + + def test_per_client_limit_spans_addresses_and_orgs( + self, monkeypatch, contacts, org_a, org_b + ): + _rate(monkeypatch, PortalLoginIPThrottle, "2/hour") + ip = "198.51.100.1" + assert _portal(org_a, "pat0@example.com", ip) == (200, 1) + assert _portal(org_a, "pat1@example.com", ip) == (200, 1) + # Neither a new address nor another org's id buys a new bucket. + assert _portal(org_a, "pat2@example.com", ip) == (429, 0) + assert _portal(org_b, "pat0@example.com", ip) == (429, 0) + assert _portal(org_a, "pat2@example.com", "198.51.100.2") == (200, 1) + assert PortalLoginToken.objects.count() == 3 + + def test_per_org_limit_holds_across_clients( + self, monkeypatch, contacts, org_a, org_b + ): + _rate(monkeypatch, PortalLoginOrgThrottle, "2/hour") + assert _portal(org_a, "pat0@example.com", "198.51.100.1") == (200, 1) + assert _portal(org_a, "pat1@example.com", "198.51.100.2") == (200, 1) + assert _portal(org_a, "pat2@example.com", "198.51.100.3") == (429, 0) + # Another org's customers are not locked out. + assert _portal(org_b, "pat0@example.com", "198.51.100.4") == (200, 1) + + def test_throttled_answer_does_not_depend_on_the_address( + self, monkeypatch, contacts, org_a + ): + _rate(monkeypatch, PortalLoginIPThrottle, "1/hour") + ip = "198.51.100.1" + assert _portal(org_a, "pat0@example.com", ip) == (200, 1) + assert _portal(org_a, "pat1@example.com", ip) == (429, 0) + assert _portal(org_a, "nobody@example.com", ip) == (429, 0) diff --git a/backend/common/throttles.py b/backend/common/throttles.py new file mode 100644 index 00000000..963103d2 --- /dev/null +++ b/backend/common/throttles.py @@ -0,0 +1,78 @@ +"""Rate limits for the anonymous sign-in request endpoints, and the mixin every +view pairing a per-client throttle with a shared one uses. + +`MagicLinkRequestView` and `PortalLoginRequestView` each mint a token and send +an email for a caller nobody has authenticated. Their own per-recipient check +(5 an hour per email or contact) caps what one inbox receives but not what one +caller sends: a fresh address is a fresh bucket. These layers sit in front of +it: + +- Per client, bucketed on `common.request_meta.client_ip`. Both web pages call + the API from the SvelteKit server, so every web visitor shares that server's + bucket unless it sends their address signed with `RELAY_SECRET`. The default + rates leave room for that, and `.env.example` says so. +- `MagicLinkGlobalThrottle`, across every caller. The magic link mails any + address (sign-in also registers), so this is the backstop for a sender spread + over many addresses. +- `PortalLoginOrgThrottle`, per org. The portal only mails an existing contact + of that org, so the per-org cap is the useful backstop there. + +All of them need the shared cache `crm/settings.py` builds from `CACHE_URL`. +""" + +from rest_framework.throttling import SimpleRateThrottle + +from common.request_meta import client_ip + + +class FirstRefusalThrottleMixin: + """Stop at the first throttle that refuses, and list the per-client one first. + + DRF's `check_throttles` consults every throttle even after one refuses, and + a `SimpleRateThrottle` records each request it allows. So a request the + per-client limit refused still counted toward the shared one, and a single + address could fill the shared bucket alone and lock every other visitor + out of signing in. + """ + + def check_throttles(self, request): + for throttle in self.get_throttles(): + if not throttle.allow_request(request, self): + self.throttled(request, throttle.wait()) + + +class _ClientIPThrottle(SimpleRateThrottle): + def get_cache_key(self, request, view): + return self.cache_format % { + "scope": self.scope, + "ident": client_ip(request) or "unknown", + } + + +class MagicLinkIPThrottle(_ClientIPThrottle): + scope = "magic_link_ip" + + +class MagicLinkGlobalThrottle(SimpleRateThrottle): + scope = "magic_link_global" + + def get_cache_key(self, request, view): + return self.cache_format % {"scope": self.scope, "ident": "all"} + + +class PortalLoginIPThrottle(_ClientIPThrottle): + """Not bucketed per org, unlike the web form limit: an org id is not secret + (it is in every portal email), so a per-org IP bucket would let one caller + multiply their allowance by the number of org ids they hold.""" + + scope = "portal_login_ip" + + +class PortalLoginOrgThrottle(SimpleRateThrottle): + scope = "portal_login_org" + + def get_cache_key(self, request, view): + return self.cache_format % { + "scope": self.scope, + "ident": str(view.kwargs.get("org_id")), + } diff --git a/backend/common/views/auth_views.py b/backend/common/views/auth_views.py index 2fd3d4c9..0fc3dacc 100644 --- a/backend/common/views/auth_views.py +++ b/backend/common/views/auth_views.py @@ -20,6 +20,11 @@ from common.models import Org, Profile, User from common.permissions import is_org_admin from common.request_meta import client_ip from common.serializer import OrgAwareRefreshToken +from common.throttles import ( + FirstRefusalThrottleMixin, + MagicLinkGlobalThrottle, + MagicLinkIPThrottle, +) from common.utils import CURRENCY_SYMBOLS logger = logging.getLogger(__name__) @@ -737,14 +742,16 @@ class OrgSwitchView(APIView): ) -class MagicLinkRequestView(APIView): +class MagicLinkRequestView(FirstRefusalThrottleMixin, APIView): """ Request a magic link for passwordless login/registration. - Always returns 200 to prevent email enumeration. + Returns 200 whatever the email, to prevent enumeration. A throttled caller + gets 429 before the email is read, so that says nothing about it either. """ permission_classes = [] authentication_classes = [] + throttle_classes = [MagicLinkIPThrottle, MagicLinkGlobalThrottle] @extend_schema( tags=["auth"], diff --git a/backend/common/views/calendar_feed_views.py b/backend/common/views/calendar_feed_views.py index f3e49c30..edd11bec 100644 --- a/backend/common/views/calendar_feed_views.py +++ b/backend/common/views/calendar_feed_views.py @@ -47,6 +47,7 @@ from common.org_time import activate_org_timezone from common.permissions import HasOrgContext from common.request_meta import client_ip from common.tasks import set_rls_context +from common.throttles import FirstRefusalThrottleMixin from tasks.access import visible_tasks_qs # What the feed carries, all owner decisions: open tasks only, due from 90 days @@ -174,7 +175,7 @@ class _AlwaysJSON(BaseContentNegotiation): return (renderers[0], renderers[0].media_type) -class PublicCalendarFeedView(APIView): +class PublicCalendarFeedView(FirstRefusalThrottleMixin, APIView): authentication_classes: list = [] permission_classes = (AllowAny,) throttle_classes = [CalendarFeedIPThrottle, CalendarFeedTokenThrottle] diff --git a/backend/common/views/portal_auth_views.py b/backend/common/views/portal_auth_views.py index ce031e88..d8b156c1 100644 --- a/backend/common/views/portal_auth_views.py +++ b/backend/common/views/portal_auth_views.py @@ -24,6 +24,11 @@ from common.models import Org, PortalLoginToken from common.portal_auth import mint_portal_token from common.request_meta import client_ip from common.tasks import set_rls_context +from common.throttles import ( + FirstRefusalThrottleMixin, + PortalLoginIPThrottle, + PortalLoginOrgThrottle, +) from contacts.models import Contact TOKEN_TTL = timedelta(minutes=10) @@ -51,11 +56,16 @@ class PortalLoginVerifySerializer(serializers.Serializer): code = serializers.CharField(max_length=6) -class PortalLoginRequestView(APIView): - """Mint and email a one-time sign-in code, if the caller is a contact here.""" +class PortalLoginRequestView(FirstRefusalThrottleMixin, APIView): + """Mint and email a one-time sign-in code, if the caller is a contact here. + + The throttles answer 429 before the org or the email is looked at, so a + throttled response reveals neither. + """ permission_classes = [] authentication_classes = [] + throttle_classes = [PortalLoginIPThrottle, PortalLoginOrgThrottle] def post(self, request, org_id): generic = Response({"message": GENERIC_MESSAGE}, status=status.HTTP_200_OK) diff --git a/backend/crm/settings.py b/backend/crm/settings.py index 39c33915..7db7cdd1 100644 --- a/backend/crm/settings.py +++ b/backend/crm/settings.py @@ -391,11 +391,11 @@ REST_FRAMEWORK = { "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", "PAGE_SIZE": 10, "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", - # Public web form submission only (issue #634). There is deliberately no - # DEFAULT_THROTTLE_CLASSES entry: setting one would rate-limit every + # Anonymous endpoints only (issue #634 started it). There is deliberately + # no DEFAULT_THROTTLE_CLASSES entry: setting one would rate-limit every # authenticated endpoint in the app, which nobody asked for and which would - # surface as intermittent 429s in the CRM UI. The two views that want these - # rates name their throttle classes explicitly. + # surface as intermittent 429s in the CRM UI. Each view that wants these + # rates names its throttle classes explicitly. "DEFAULT_THROTTLE_RATES": { "webform_submit_ip": os.environ.get("WEBFORM_THROTTLE_IP", "10/hour"), "webform_submit_global": os.environ.get("WEBFORM_THROTTLE_GLOBAL", "200/day"), @@ -415,6 +415,20 @@ REST_FRAMEWORK = { "calendar_feed_token": os.environ.get( "CALENDAR_FEED_THROTTLE_TOKEN", "60/hour" ), + # Magic-link and portal sign-in requests, per address + # (`common.throttles`). One person needs a handful; the rest is room + # for an office behind one NAT, and for a deployment without + # RELAY_SECRET, where every web sign-in shares the SvelteKit server's + # bucket. + "magic_link_ip": os.environ.get("MAGIC_LINK_THROTTLE_IP", "30/hour"), + "portal_login_ip": os.environ.get("PORTAL_LOGIN_THROTTLE_IP", "30/hour"), + # Magic-link requests across every caller. The backstop against a + # sender spread over many addresses using us to mail arbitrary inboxes. + # Reaching it pauses sign-in email for everyone, so it sits far above + # real sign-in traffic. + "magic_link_global": os.environ.get("MAGIC_LINK_THROTTLE_GLOBAL", "300/hour"), + # Portal sign-in requests per org, across every caller. + "portal_login_org": os.environ.get("PORTAL_LOGIN_THROTTLE_ORG", "100/hour"), }, } @@ -448,7 +462,7 @@ else: # Shared secret between this API and the SvelteKit servers that relay anonymous -# visitors to it: the web app's help center and estimate portal, and the +# visitors to it: the web app's sign-in pages, help center and estimate portal, and the # marketing site's contact form. A request carrying it in # X-BottleCRM-Relay-Secret is believed about the visitor it names in # X-BottleCRM-Client-IP (`common.request_meta.client_ip`), so each visitor gets diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1b03d217..d1c6fdb3 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "django-crm" -version = "1.14.4" +version = "1.14.5" # The one-line summary PyPI shows under the project title and in search results, # so it carries the terms people actually search for ("open source CRM", # "self-hosted CRM", "Django CRM") and names the modules rather than the diff --git a/backend/uv.lock b/backend/uv.lock index 712a7ece..fae63688 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -681,7 +681,7 @@ wheels = [ [[package]] name = "django-crm" -version = "1.14.4" +version = "1.14.5" source = { editable = "." } dependencies = [ { name = "cairocffi" }, diff --git a/backend/webforms/public_views.py b/backend/webforms/public_views.py index 4eef13e0..db26a0a4 100644 --- a/backend/webforms/public_views.py +++ b/backend/webforms/public_views.py @@ -36,6 +36,7 @@ from common.links import api_url from common.org_time import activate_org_timezone from common.request_meta import client_ip, referer from common.tasks import set_rls_context +from common.throttles import FirstRefusalThrottleMixin from webforms import captcha from webforms.dynamic_serializer import HONEYPOT_FIELD, build_serializer from webforms.models import WebForm, WebFormDailyStat, WebFormSubmission @@ -130,7 +131,7 @@ class PublicWebFormMixin: } -class WebFormSubmitView(PublicWebFormMixin, APIView): +class WebFormSubmitView(FirstRefusalThrottleMixin, PublicWebFormMixin, APIView): throttle_classes = [WebFormIPThrottle, WebFormGlobalThrottle] def post(self, request, org_id, form_id): diff --git a/backend/webforms/tests/test_submit.py b/backend/webforms/tests/test_submit.py index 874cf396..6d214c38 100644 --- a/backend/webforms/tests/test_submit.py +++ b/backend/webforms/tests/test_submit.py @@ -310,6 +310,38 @@ class TestThrottling: ) assert response.status_code == 429 + def test_a_refused_client_does_not_spend_the_forms_daily_cap( + self, unauthenticated_client, org_a, form, tight_rates + ): + """DRF consults every throttle even after one refuses, so without + FirstRefusalThrottleMixin one address flooding the form filled its + daily cap and closed it to every other visitor.""" + flood = [ + unauthenticated_client.post( + submit_url(org_a, form), + {"email": f"flood{index}@example.com"}, + format="json", + REMOTE_ADDR="198.51.100.1", + ).status_code + for index in range(5) + ] + assert flood == [200, 200, 429, 429, 429] + visitor = unauthenticated_client.post( + submit_url(org_a, form), + {"email": "real@example.com"}, + format="json", + REMOTE_ADDR="198.51.100.2", + ) + assert visitor.status_code == 200 + # The daily cap (3) still binds. + late = unauthenticated_client.post( + submit_url(org_a, form), + {"email": "late@example.com"}, + format="json", + REMOTE_ADDR="198.51.100.3", + ) + assert late.status_code == 429 + def test_the_limit_is_per_form_not_per_org( self, unauthenticated_client, org_a, form, admin_profile, tight_rates ): diff --git a/backend/webforms/throttles.py b/backend/webforms/throttles.py index ee65729d..d0232f07 100644 --- a/backend/webforms/throttles.py +++ b/backend/webforms/throttles.py @@ -7,7 +7,10 @@ Two layers, because neither is sufficient alone: rotating the header does not buy a fresh bucket. It stops one visitor or one bot from flooding. - `WebFormGlobalThrottle` caps submissions per form across all clients, the - backstop for a sender spread across many real addresses. + backstop for a sender spread across many real addresses. The view checks + them in that order and stops at the first refusal + (`common.throttles.FirstRefusalThrottleMixin`), so a client the per-IP limit + refused cannot fill the form's daily cap alone. Both need a shared cache to mean anything across workers. `crm/settings.py` configures one from `CACHE_URL` and says what happens when it is not set. diff --git a/docs/api/authentication.md b/docs/api/authentication.md index f9524eda..ac594e45 100644 --- a/docs/api/authentication.md +++ b/docs/api/authentication.md @@ -112,6 +112,11 @@ This is deliberate, see [First sign-in](../getting-started/first-sign-in.md#magi means a client cannot distinguish "sent" from "rejected" from this response; there is no `400` to handle here at all. A generated token expires after 10 minutes. +The one other answer is `429`, from the per-client and site-wide limits in +`backend/common/throttles.py` (`MAGIC_LINK_THROTTLE_IP`, default `30/hour`, and +`MAGIC_LINK_THROTTLE_GLOBAL`, default `300/hour`). They run before the email is read, so a `429` +says nothing about the address either. + `POST /api/auth/magic-link/verify/` (`MagicLinkVerifyView`, `backend/common/views/auth_views.py:658-768`) exchanges the token from a link-delivery email: diff --git a/docs/api/customer-portal.md b/docs/api/customer-portal.md index 1b9b3de9..5a42ed87 100644 --- a/docs/api/customer-portal.md +++ b/docs/api/customer-portal.md @@ -44,7 +44,10 @@ validation, or was rate limited: {"message": "If this email is valid, you will receive a sign-in code."} ``` -There is no `400` to handle. The uniformity is the point: any difference in status, body, or +There is no `400` to handle. The one other answer is `429`, from the per-client and per-org limits +in `backend/common/throttles.py` (`PORTAL_LOGIN_THROTTLE_IP`, default `30/hour`, and +`PORTAL_LOGIN_THROTTLE_ORG`, default `100/hour`); they run before the org or the email is looked at, +so it reveals neither. The uniformity is the point: any difference in status, body, or observable work would let a stranger test which of their competitors' customers you do business with. A six digit code is emailed, stored only as a PBKDF2 hash, and expires after 10 minutes. Five codes per contact per hour, matching the internal magic-link limit. Each new request retires diff --git a/docs/api/errors.md b/docs/api/errors.md index 43cb6066..d1208b32 100644 --- a/docs/api/errors.md +++ b/docs/api/errors.md @@ -24,12 +24,12 @@ resource that doesn't exist, or that exists but the caller must not be able to d existing. See [Not found versus forbidden](#not-found-versus-forbidden); `502` where a view depends on an external service and that service is unreachable, for example the Google token exchange in [Google OAuth](authentication.md#google-oauth-web) (`backend/common/views/auth_views.py:110-114`). -There is no application-level `429`; `REST_FRAMEWORK` sets no `DEFAULT_THROTTLE_CLASSES`, so DRF's -built-in throttling is not in play anywhere. The one place this codebase does rate-limit -(`POST /api/auth/magic-link/request/`, five tokens per address per hour) enforces it by silently -returning its normal `200` rather than a `429`, specifically so the response can't be used to -distinguish "rate limited" from "sent" from "invalid address". See -[Magic links](authentication.md#magic-links). +`429` comes only from the anonymous endpoints that name their own throttles: web form submit, the +public help center, the calendar feed, and the magic-link and portal sign-in requests. +`REST_FRAMEWORK` sets no `DEFAULT_THROTTLE_CLASSES`, so no authenticated endpoint returns it. The +sign-in requests also cap tokens per address (five an hour), and that limit answers with their +normal `200` rather than a `429`, so the response can't be used to distinguish "rate limited" from +"sent" from "invalid address". See [Magic links](authentication.md#magic-links). ## Validation errors diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 13e1723f..f37bb0c2 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -54,8 +54,8 @@ itself, so they aren't in the table below: | `DEBUG` | `False` | No | Django debug mode. Also the sole gate on the `devlogin` management command. See [Management commands](management-commands.md#devlogin). | | `ALLOWED_HOSTS` | `localhost,127.0.0.1` | Yes, once serving traffic under a real hostname | Comma-separated list, split on `,` into Django's `ALLOWED_HOSTS`. | | `TRUST_PROXY_SSL_HEADER` | `False` | Yes, if TLS is terminated by a proxy in front of Django | Set to `true` (case-insensitive) to set Django's `SECURE_PROXY_SSL_HEADER` to `("HTTP_X_FORWARDED_PROTO", "https")`. Without it, a request arriving from nginx or an ALB over plain HTTP makes `request.is_secure()` false, and `SecurityMiddleware` then emits **no** `Strict-Transport-Security` header at all, so the `SECURE_HSTS_SECONDS` / `INCLUDE_SUBDOMAINS` / `PRELOAD` settings are silently inert. Only turn it on when that proxy is the sole route to the app **and** it overwrites `X-Forwarded-Proto` rather than passing through a client-supplied one, because otherwise any caller reaching Django directly can make `request.is_secure()` return true. The http-to-https redirect belongs on the proxy; `SECURE_SSL_REDIRECT` is deliberately not set. | -| `NUM_PROXIES` | unset | Only when `REMOTE_ADDR` is the proxy (gunicorn behind nginx: `1`). Leave unset under uvicorn `--proxy-headers` | How many proxies in front of Django append to `X-Forwarded-For` (nginx with `$proxy_add_x_forwarded_for` is one). Sets DRF's `NUM_PROXIES`, read by `common.request_meta.client_ip`, which supplies every recorded client IP (security audit log, web form submissions, estimate acceptance, magic-link and portal sign-in tokens) and every per-IP throttle (web form submit, help center). It uses the entry that many places from the right, the address the outermost proxy saw; entries to its left are whatever the caller sent and are ignored, so rotating the header neither picks a throttle bucket nor forges a recorded address. Unset or `0` uses `REMOTE_ADDR`. That is correct when the app server already resolves the client from trusted proxies, as uvicorn does with `--proxy-headers --forwarded-allow-ips=127.0.0.1`; setting `NUM_PROXIES` on top of that counts the proxy twice. Either way a request a SvelteKit server relays for a visitor arrives from the relay; `RELAY_SECRET` is what tells those visitors apart. When `REMOTE_ADDR` is the proxy itself (gunicorn behind nginx), unset means every visitor shares one throttle bucket, so set it to the number of proxies. Set it higher than the real number of proxies and callers can choose the IP again. | -| `RELAY_SECRET` | unset | Recommended wherever a SvelteKit server relays anonymous visitors (the web app's help center and estimate portal, the marketing site's contact form) | Shared secret with those SvelteKit servers. A request whose `X-BottleCRM-Relay-Secret` header matches it (constant-time compare) is believed about the visitor address in `X-BottleCRM-Client-IP`, so each visitor gets their own per-IP throttle bucket and recorded address instead of the relay's. Without a match both headers are ignored and `NUM_PROXIES` decides as above; a visitor address that is not an IP falls back the same way. Set the same value as `RELAY_SECRET` in the web app's environment (private, never `PUBLIC_`), and run that app with `ADDRESS_HEADER=X-Forwarded-For` and `XFF_DEPTH=1` behind nginx so `getClientAddress()` is the visitor rather than nginx, and with `HOST=127.0.0.1` so it cannot be reached around nginx with a forged `X-Forwarded-For`. The relays also send the visitor in `X-Forwarded-For` for APIs before 1.12.0, which read its first entry; this release ignores that header unless `NUM_PROXIES` is set. At least 32 characters: `crm/settings.py` refuses to start with a shorter value, and `client_ip` never matches one. Generate with `python -c "import secrets; print(secrets.token_urlsafe(48))"`. Anyone holding it can choose the address they are throttled and recorded as, so treat it like a password; its header is scrubbed from Sentry events. | +| `NUM_PROXIES` | unset | Only when `REMOTE_ADDR` is the proxy (gunicorn behind nginx: `1`). Leave unset under uvicorn `--proxy-headers` | How many proxies in front of Django append to `X-Forwarded-For` (nginx with `$proxy_add_x_forwarded_for` is one). Sets DRF's `NUM_PROXIES`, read by `common.request_meta.client_ip`, which supplies every recorded client IP (security audit log, web form submissions, estimate acceptance, magic-link and portal sign-in tokens) and every per-IP throttle (web form submit, help center, calendar feed, magic-link and portal sign-in). It uses the entry that many places from the right, the address the outermost proxy saw; entries to its left are whatever the caller sent and are ignored, so rotating the header neither picks a throttle bucket nor forges a recorded address. Unset or `0` uses `REMOTE_ADDR`. That is correct when the app server already resolves the client from trusted proxies, as uvicorn does with `--proxy-headers --forwarded-allow-ips=127.0.0.1`; setting `NUM_PROXIES` on top of that counts the proxy twice. Either way a request a SvelteKit server relays for a visitor arrives from the relay; `RELAY_SECRET` is what tells those visitors apart. When `REMOTE_ADDR` is the proxy itself (gunicorn behind nginx), unset means every visitor shares one throttle bucket, so set it to the number of proxies. Set it higher than the real number of proxies and callers can choose the IP again. | +| `RELAY_SECRET` | unset | Recommended wherever a SvelteKit server relays anonymous visitors (the web app's sign-in pages, help center and estimate portal, the marketing site's contact form) | Shared secret with those SvelteKit servers. A request whose `X-BottleCRM-Relay-Secret` header matches it (constant-time compare) is believed about the visitor address in `X-BottleCRM-Client-IP`, so each visitor gets their own per-IP throttle bucket and recorded address instead of the relay's. Without a match both headers are ignored and `NUM_PROXIES` decides as above; a visitor address that is not an IP falls back the same way. Set the same value as `RELAY_SECRET` in the web app's environment (private, never `PUBLIC_`), and run that app with `ADDRESS_HEADER=X-Forwarded-For` and `XFF_DEPTH=1` behind nginx so `getClientAddress()` is the visitor rather than nginx, and with `HOST=127.0.0.1` so it cannot be reached around nginx with a forged `X-Forwarded-For`. The relays also send the visitor in `X-Forwarded-For` for APIs before 1.12.0, which read its first entry; this release ignores that header unless `NUM_PROXIES` is set. At least 32 characters: `crm/settings.py` refuses to start with a shorter value, and `client_ip` never matches one. Generate with `python -c "import secrets; print(secrets.token_urlsafe(48))"`. Anyone holding it can choose the address they are throttled and recorded as, so treat it like a password; its header is scrubbed from Sentry events. | | `DJANGO_ORG_API_KEY_AUTH` | `true` | No | Set to `false`, `0`, `no` or `off` to refuse the organization API key (`Token: `) as an authentication method entirely, on every endpoint. The key is one non-expiring credential per tenant that resolves to an arbitrary active ADMIN, so it cannot be revoked per-integration and it reads every record in the org. It is read-only and barred from the credential endpoints even when enabled (see [Tokens and API keys](../api/tokens-and-api-keys.md)), but a deployment whose integrations have all moved to personal access tokens should turn it off. Left on by default so an upgrade breaks nothing. | ### Database @@ -109,6 +109,10 @@ Pooling uses psycopg 3's `psycopg_pool` through Django's PostgreSQL backend. It | `HELP_CENTER_THROTTLE_IP` | `600/hour` | No | Rate limit for the public help center API (`/api/public/help/`), per client IP. With `RELAY_SECRET` set on the API and the web app, the web app sends each visitor's address signed, so each reader and crawler gets its own bucket; without it they all share the web app's. The IP is `common.request_meta.client_ip`; the pages it protects are public by the org's choice. | | `HELP_CENTER_THROTTLE_GLOBAL` | `10000/hour` | No | Rate limit for the public help center API, per help center across all clients. No number of addresses evades this one, so it is the backstop behind `HELP_CENTER_THROTTLE_IP`. It is bucketed per help center, so traffic to one org's pages never limits another's. Raise it for a busy help center. | | `CALENDAR_FEED_THROTTLE_IP` | `1000/hour` | No | Rate limit for the task calendar feed (`/api/public/calendar/.ics`), per client IP. Generous because Google, Microsoft and Apple fetch subscribed calendars from shared addresses that carry many users' feeds at once. The IP is `common.request_meta.client_ip`. | +| `MAGIC_LINK_THROTTLE_IP` | `30/hour` | No | Rate limit for magic-link sign-in requests (`/api/auth/magic-link/request/`), per client IP. The web sign-in page calls the API from its server, so without `RELAY_SECRET` every web sign-in shares one bucket; the default leaves room for that and for an office behind one NAT. Each address is separately capped at five links an hour. | +| `MAGIC_LINK_THROTTLE_GLOBAL` | `300/hour` | No | Rate limit for magic-link sign-in requests across all clients, the backstop against a sender spread over many addresses using the instance to mail arbitrary inboxes. Reaching it pauses sign-in email for everyone until the hour rolls over, so keep it well above real sign-in traffic. | +| `PORTAL_LOGIN_THROTTLE_IP` | `30/hour` | No | Rate limit for customer portal sign-in code requests (`/api/portal/login//request/`), per client IP across all orgs. Same `RELAY_SECRET` caveat as `MAGIC_LINK_THROTTLE_IP`. | +| `PORTAL_LOGIN_THROTTLE_ORG` | `100/hour` | No | Rate limit for customer portal sign-in code requests, per org across all clients. Reaching it pauses portal sign-in for that org only. | | `CALENDAR_FEED_THROTTLE_TOKEN` | `60/hour` | No | Rate limit for the task calendar feed, per feed URL across all clients. Bucketed on the token's hash, never the raw token. A calendar app polls every few minutes at most, so this is only reached by something hammering one URL. | ### CORS and CSRF