diff --git a/HISTORY.rst b/HISTORY.rst index 75e7a307..adffb280 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,17 @@ This project adheres to `Semantic Versioning `_. Unreleased =========================== +* **Security fix.** Query parameters (``$$foo$$``) were interpolated into the SQL as raw text. Because + parameter values are taken from the request (``?params=foo:value``, the documented way to share a + parameterized query), anyone who could get a user to open a link could inject arbitrary SQL into a + saved query, including queries they had no permission to edit. Parameters are now passed to the + database as bound parameters and never become part of the SQL text. + + This changes what a parameter can be: values are now always data, so a query that relied on a + parameter carrying a SQL fragment (for example ``ORDER BY $$sort$$`` or ``LIMIT $$n$$`` written as a + bare token) must be edited to use a real column or value. Numeric-looking values are still bound + numerically, so ``WHERE id = $$id$$`` continues to compare numbers. + * `#722`_: The query list (home page) no longer loads the entire query log history on every view. Run counts and last runs are computed in SQL, so the page stays fast as logs accumulate (300k logs: ~9.7s -> ~0.9s). The query list and query log pages also no longer run an extra database query per row. diff --git a/explorer/models.py b/explorer/models.py index 67939df8..b26e47b0 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -3,7 +3,7 @@ import uuid from django.conf import settings -from django.core.exceptions import ValidationError +from django.core.exceptions import ImproperlyConfigured, ValidationError from django.db import DatabaseError, models, transaction from django.urls import reverse from django.utils.translation import gettext_lazy as _ @@ -94,7 +94,7 @@ def passes_blacklist(self): return passes_blacklist(self.final_sql()) def final_sql(self): - return swap_params(self.sql, self.available_params()) + return swap_params(self.sql, self.available_params())[0] def execute_query_only(self): # check blacklist every time sql is run to catch parameterized SQL @@ -110,7 +110,7 @@ def execute_query_only(self): ) conn = self.database_connection or default_db_connection() return QueryResult( - self.final_sql(), conn.as_django_connection() + self.sql, conn.as_django_connection(), params=self.available_params() ) def execute_with_logging(self, executing_user): @@ -266,12 +266,62 @@ class Meta: unique_together = ["query", "user"] +def _placeholder_for(connection): + """ + A bind placeholder for the given connection's DB-API paramstyle. + + Values are positional, so the placeholder only depends on how many came before it. + """ + paramstyle = getattr(connection, "paramstyle", None) or "format" + if paramstyle == "qmark": + return lambda n: "?" + if paramstyle in ("format", "pyformat"): + return lambda n: "%s" + if paramstyle == "numeric": + return lambda n: f":{n}" + raise ImproperlyConfigured( + f"Explorer does not support the '{paramstyle}' paramstyle. " + f"Refusing to interpolate query parameters into SQL." + ) + + +def _escape_literal_percents(sql, paramstyle): + """ + Escape literal ``%`` in SQL that is about to be executed with bound parameters. + + Drivers using "format"/"pyformat" run the query through ``%`` interpolation once values + are supplied, so a literal ``%`` in the query text (the common ``LIKE '%foo%'``) would + otherwise be read as a placeholder. Without bound values the driver passes the SQL + through untouched, which is why this is only applied when binding happens. + """ + if paramstyle not in ("format", "pyformat"): + return sql + out = [] + ix = 0 + while ix < len(sql): + char = sql[ix] + if char == "%": + following = sql[ix + 1:ix + 2] + # Leave the placeholders we just emitted alone. + if following == "s": + out.append("%s") + ix += 2 + continue + out.append("%%") + ix += 1 + continue + out.append(char) + ix += 1 + return "".join(out) + + class QueryResult: - def __init__(self, sql, connection): + def __init__(self, sql, connection, params=None): self.sql = sql self.connection = connection + self.params = params cursor, duration = self.execute_query() @@ -348,9 +398,14 @@ def execute_query(self): cursor = self.connection.cursor() start_time = time() + placeholder = _placeholder_for(self.connection) + sql, bound = swap_params(self.sql, self.params, bind=placeholder) + if bound: + sql = _escape_literal_percents(sql, getattr(self.connection, "paramstyle", "format")) + try: with transaction.atomic(self.connection.alias): - cursor.execute(self.sql) + cursor.execute(sql, bound or None) except DatabaseError as e: cursor.close() raise e diff --git a/explorer/tests/test_models.py b/explorer/tests/test_models.py index 04591ac9..2728f114 100644 --- a/explorer/tests/test_models.py +++ b/explorer/tests/test_models.py @@ -138,6 +138,43 @@ def test_final_sql_fails_blacklist_with_bad_param(self): with self.assertRaises(ValidationError): q.execute_query_only() + def test_quoted_param_is_bound_not_interpolated(self): + q = SimpleQueryFactory(sql="select '$$who$$' as who;") + q.params = {"who": "x' OR '1'='1"} + qr = q.execute_query_only() + self.assertEqual(qr.data, [["x' OR '1'='1"]]) + + def test_bare_param_is_bound_not_interpolated(self): + q = SimpleQueryFactory(sql="select 1 where 'a' = $$v$$;") + q.params = {"v": "'a' union select name from sqlite_master --"} + self.assertEqual(q.execute_query_only().data, []) + + def test_param_is_inert_even_when_blacklist_does_not_catch_it(self): + # /*!...*/ is executable in MySQL but the blacklist does not see the keyword, so this + # asserts the bind, not the blacklist, is what keeps the value out of the SQL. + payload = "x'; /*!50000DELETE*/ FROM explorer_query; -- " + q = SimpleQueryFactory(sql="select '$$v$$' as v;") + q.params = {"v": payload} + self.assertTrue(q.passes_blacklist()[0]) + self.assertEqual(q.execute_query_only().data, [[payload]]) + self.assertTrue(Query.objects.exists()) + + def test_numeric_param_is_bound_with_its_original_type(self): + q = SimpleQueryFactory(sql="select 6870+4 where 1=$$n:1$$;") + self.assertEqual(q.execute_query_only().data, [[6874]]) + + def test_escape_literal_percents_leaves_placeholders_intact(self): + from explorer.models import _escape_literal_percents + self.assertEqual( + _escape_literal_percents("select * from t where n like '%a%' and id = %s", "format"), + "select * from t where n like '%%a%%' and id = %s" + ) + # qmark drivers never interpolate, so their SQL is left alone. + self.assertEqual( + _escape_literal_percents("select '100%' where 1 = ?", "qmark"), + "select '100%' where 1 = ?" + ) + def test_query_will_execute_with_null_database_connection(self): q = SimpleQueryFactory(sql="select 1;") q.database_connection_id = None diff --git a/explorer/tests/test_utils.py b/explorer/tests/test_utils.py index e3f05523..361886c1 100644 --- a/explorer/tests/test_utils.py +++ b/explorer/tests/test_utils.py @@ -193,21 +193,46 @@ def test_params_get_swapped(self): sql = "please Swap $$this$$ and $$THat$$" expected = "please Swap here and there" params = {"this": "here", "that": "there"} - got = swap_params(sql, params) + got, bound = swap_params(sql, params) self.assertEqual(got, expected) + self.assertEqual(bound, []) def test_empty_params_does_nothing(self): sql = "please swap $$this$$ and $$that$$" params = None - got = swap_params(sql, params) + got, bound = swap_params(sql, params) self.assertEqual(got, sql) + self.assertEqual(bound, []) def test_non_string_param_gets_swapper(self): sql = "please swap $$this$$" expected = "please swap 1" params = {"this": 1} - got = swap_params(sql, params) + got, bound = swap_params(sql, params) self.assertEqual(got, expected) + self.assertEqual(bound, []) + + def test_quoted_param_becomes_bare_placeholder(self): + got, bound = swap_params("select '$$foo$$'", {"foo": "bar"}, bind=lambda n: "?") + self.assertEqual(got, "select ?") + self.assertEqual(bound, ["bar"]) + + def test_bare_param_becomes_placeholder(self): + got, bound = swap_params("select * from t limit $$n$$", {"n": "5"}, bind=lambda n: "?") + self.assertEqual(got, "select * from t limit ?") + self.assertEqual(bound, [5]) + + def test_bind_placeholders_are_positional(self): + got, bound = swap_params( + "$$a$$, $$b$$", {"a": "1", "b": "2"}, bind=lambda n: f":{n}" + ) + self.assertEqual(got, ":1, :2") + self.assertEqual(bound, [1, 2]) + + def test_bind_does_not_treat_value_as_replacement_template(self): + got, bound = swap_params("select '$$p$$'", {"p": r"C:\path\1"}, bind=lambda n: "?") + self.assertEqual(got, "select ?") + self.assertEqual(bound, [r"C:\path\1"]) def _assertSwap(self, tuple): self.assertEqual(extract_params(tuple[0]), tuple[1]) diff --git a/explorer/tests/test_views.py b/explorer/tests/test_views.py index 87d94354..f680e077 100644 --- a/explorer/tests/test_views.py +++ b/explorer/tests/test_views.py @@ -4,6 +4,7 @@ from datetime import timedelta import unittest import os +from urllib.parse import quote from unittest.mock import Mock, patch, MagicMock from unittest import skipIf @@ -458,7 +459,17 @@ def test_params_in_download(self): "foo:123" ) resp = self.client.get(url) - self.assertContains(resp, "'123'") + self.assertContains(resp, "123") + + def test_params_in_url_cannot_inject_sql(self): + # Regression: parameter values come from the URL, so they must be bound, not interpolated. + q = SimpleQueryFactory(sql="select '$$who$$' as who;") + url = "{}?params={}".format( + reverse("download_query", kwargs={"query_id": q.id}), + quote("who:') union select 1 where '1'='1", safe="") + ) + resp = self.client.get(url) + self.assertNotContains(resp, "django_migrations") def test_download_defaults_to_csv(self): query = SimpleQueryFactory() diff --git a/explorer/utils.py b/explorer/utils.py index a3454640..014d3f02 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -61,13 +61,56 @@ def param(name): return f"{EXPLORER_PARAM_TOKEN}{name}{EXPLORER_PARAM_TOKEN}" -def swap_params(sql, params): +def _param_regex(name, placeholder=None): + token = rf"\$\${re.escape(str(name))}(?:\|([^\$\:]+))?(?:\:([^\$]+))?\$\$" + if placeholder is None: + return re.compile(token, re.I) + # A parameter written as '$$id$$' (quoted, as the docs recommend) has to become a bind + # placeholder *without* its quotes, otherwise the driver sees the literal string "?". + return re.compile(rf"""(['"]){token}\1|{token}""", re.I) + + +def _bindable(value): + """ + Coerce a parameter value to the type it would have had as literal SQL. + + Parameters arrive as strings from the URL. Binding them as strings would make + ``WHERE id = $$id$$`` compare a number to text, so numeric-looking values are bound + numerically instead. + """ + if isinstance(value, str): + for converter in (int, float): + try: + return converter(value) + except ValueError: + continue + return value + + +def swap_params(sql, params, bind=None): + """ + Substitute ``$$name$$`` tokens in ``sql``. + + With no ``bind`` argument the values are interpolated straight into the SQL text. That is + only safe for display (logging, the blacklist check). Values come from the request + (``?params=name:value``), so anything that will be *executed* must pass ``bind``. + + :param bind: callable taking the number of values bound so far and returning that driver's + bind placeholder. When given, values are collected instead of interpolated. + :return: ``(sql, bound_values)``; ``bound_values`` is empty when ``bind`` is None. + """ p = params.items() if params else {} + bound = [] + + def replace(match, value): + if bind is None: + return str(value) + bound.append(_bindable(value)) + return bind(len(bound)) + for k, v in p: - fmt_k = re.escape(str(k).lower()) - regex = re.compile(rf"\$\${fmt_k}(?:\|([^\$\:]+))?(?:\:([^\$]+))?\$\$", re.I) - sql = regex.sub(str(v), sql) - return sql + sql = _param_regex(k, bind).sub(lambda m, v=v: replace(m, v), sql) + return sql, bound def extract_params(text):