Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,17 @@ This project adheres to `Semantic Versioning <https://semver.org/>`_.

Unreleased
===========================
* **Security fix.** Query parameters (``$$foo$$``) were interpolated into the SQL as raw text. Because
parameter values are taken from the request (``?params=foo:value``, the documented way to share a
parameterized query), anyone who could get a user to open a link could inject arbitrary SQL into a
saved query, including queries they had no permission to edit. Parameters are now passed to the
database as bound parameters and never become part of the SQL text.

This changes what a parameter can be: values are now always data, so a query that relied on a
parameter carrying a SQL fragment (for example ``ORDER BY $$sort$$`` or ``LIMIT $$n$$`` written as a
bare token) must be edited to use a real column or value. Numeric-looking values are still bound
numerically, so ``WHERE id = $$id$$`` continues to compare numbers.

* `#722`_: The query list (home page) no longer loads the entire query log history on every view. Run counts and
last runs are computed in SQL, so the page stays fast as logs accumulate (300k logs: ~9.7s -> ~0.9s). The query
list and query log pages also no longer run an extra database query per row.
Expand Down
65 changes: 60 additions & 5 deletions explorer/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import uuid

from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.db import DatabaseError, models, transaction
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
Expand Down Expand Up @@ -94,7 +94,7 @@ def passes_blacklist(self):
return passes_blacklist(self.final_sql())

def final_sql(self):
return swap_params(self.sql, self.available_params())
return swap_params(self.sql, self.available_params())[0]

def execute_query_only(self):
# check blacklist every time sql is run to catch parameterized SQL
Expand All @@ -110,7 +110,7 @@ def execute_query_only(self):
)
conn = self.database_connection or default_db_connection()
return QueryResult(
self.final_sql(), conn.as_django_connection()
self.sql, conn.as_django_connection(), params=self.available_params()
)

def execute_with_logging(self, executing_user):
Expand Down Expand Up @@ -266,12 +266,62 @@ class Meta:
unique_together = ["query", "user"]


def _placeholder_for(connection):
"""
A bind placeholder for the given connection's DB-API paramstyle.

Values are positional, so the placeholder only depends on how many came before it.
"""
paramstyle = getattr(connection, "paramstyle", None) or "format"
if paramstyle == "qmark":
return lambda n: "?"
if paramstyle in ("format", "pyformat"):
return lambda n: "%s"
if paramstyle == "numeric":
return lambda n: f":{n}"
raise ImproperlyConfigured(
f"Explorer does not support the '{paramstyle}' paramstyle. "
f"Refusing to interpolate query parameters into SQL."
)


def _escape_literal_percents(sql, paramstyle):
"""
Escape literal ``%`` in SQL that is about to be executed with bound parameters.

Drivers using "format"/"pyformat" run the query through ``%`` interpolation once values
are supplied, so a literal ``%`` in the query text (the common ``LIKE '%foo%'``) would
otherwise be read as a placeholder. Without bound values the driver passes the SQL
through untouched, which is why this is only applied when binding happens.
"""
if paramstyle not in ("format", "pyformat"):
return sql
out = []
ix = 0
while ix < len(sql):
char = sql[ix]
if char == "%":
following = sql[ix + 1:ix + 2]
# Leave the placeholders we just emitted alone.
if following == "s":
out.append("%s")
ix += 2
continue
out.append("%%")
ix += 1
continue
out.append(char)
ix += 1
return "".join(out)


class QueryResult:

def __init__(self, sql, connection):
def __init__(self, sql, connection, params=None):

self.sql = sql
self.connection = connection
self.params = params

cursor, duration = self.execute_query()

Expand Down Expand Up @@ -348,9 +398,14 @@ def execute_query(self):
cursor = self.connection.cursor()
start_time = time()

placeholder = _placeholder_for(self.connection)
sql, bound = swap_params(self.sql, self.params, bind=placeholder)
if bound:
sql = _escape_literal_percents(sql, getattr(self.connection, "paramstyle", "format"))

try:
with transaction.atomic(self.connection.alias):
cursor.execute(self.sql)
cursor.execute(sql, bound or None)
except DatabaseError as e:
cursor.close()
raise e
Expand Down
37 changes: 37 additions & 0 deletions explorer/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,43 @@ def test_final_sql_fails_blacklist_with_bad_param(self):
with self.assertRaises(ValidationError):
q.execute_query_only()

def test_quoted_param_is_bound_not_interpolated(self):
q = SimpleQueryFactory(sql="select '$$who$$' as who;")
q.params = {"who": "x' OR '1'='1"}
qr = q.execute_query_only()
self.assertEqual(qr.data, [["x' OR '1'='1"]])

def test_bare_param_is_bound_not_interpolated(self):
q = SimpleQueryFactory(sql="select 1 where 'a' = $$v$$;")
q.params = {"v": "'a' union select name from sqlite_master --"}
self.assertEqual(q.execute_query_only().data, [])

def test_param_is_inert_even_when_blacklist_does_not_catch_it(self):
# /*!...*/ is executable in MySQL but the blacklist does not see the keyword, so this
# asserts the bind, not the blacklist, is what keeps the value out of the SQL.
payload = "x'; /*!50000DELETE*/ FROM explorer_query; -- "
q = SimpleQueryFactory(sql="select '$$v$$' as v;")
q.params = {"v": payload}
self.assertTrue(q.passes_blacklist()[0])
self.assertEqual(q.execute_query_only().data, [[payload]])
self.assertTrue(Query.objects.exists())

def test_numeric_param_is_bound_with_its_original_type(self):
q = SimpleQueryFactory(sql="select 6870+4 where 1=$$n:1$$;")
self.assertEqual(q.execute_query_only().data, [[6874]])

def test_escape_literal_percents_leaves_placeholders_intact(self):
from explorer.models import _escape_literal_percents
self.assertEqual(
_escape_literal_percents("select * from t where n like '%a%' and id = %s", "format"),
"select * from t where n like '%%a%%' and id = %s"
)
# qmark drivers never interpolate, so their SQL is left alone.
self.assertEqual(
_escape_literal_percents("select '100%' where 1 = ?", "qmark"),
"select '100%' where 1 = ?"
)

def test_query_will_execute_with_null_database_connection(self):
q = SimpleQueryFactory(sql="select 1;")
q.database_connection_id = None
Expand Down
31 changes: 28 additions & 3 deletions explorer/tests/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,21 +193,46 @@ def test_params_get_swapped(self):
sql = "please Swap $$this$$ and $$THat$$"
expected = "please Swap here and there"
params = {"this": "here", "that": "there"}
got = swap_params(sql, params)
got, bound = swap_params(sql, params)
self.assertEqual(got, expected)
self.assertEqual(bound, [])

def test_empty_params_does_nothing(self):
sql = "please swap $$this$$ and $$that$$"
params = None
got = swap_params(sql, params)
got, bound = swap_params(sql, params)
self.assertEqual(got, sql)
self.assertEqual(bound, [])

def test_non_string_param_gets_swapper(self):
sql = "please swap $$this$$"
expected = "please swap 1"
params = {"this": 1}
got = swap_params(sql, params)
got, bound = swap_params(sql, params)
self.assertEqual(got, expected)
self.assertEqual(bound, [])

def test_quoted_param_becomes_bare_placeholder(self):
got, bound = swap_params("select '$$foo$$'", {"foo": "bar"}, bind=lambda n: "?")
self.assertEqual(got, "select ?")
self.assertEqual(bound, ["bar"])

def test_bare_param_becomes_placeholder(self):
got, bound = swap_params("select * from t limit $$n$$", {"n": "5"}, bind=lambda n: "?")
self.assertEqual(got, "select * from t limit ?")
self.assertEqual(bound, [5])

def test_bind_placeholders_are_positional(self):
got, bound = swap_params(
"$$a$$, $$b$$", {"a": "1", "b": "2"}, bind=lambda n: f":{n}"
)
self.assertEqual(got, ":1, :2")
self.assertEqual(bound, [1, 2])

def test_bind_does_not_treat_value_as_replacement_template(self):
got, bound = swap_params("select '$$p$$'", {"p": r"C:\path\1"}, bind=lambda n: "?")
self.assertEqual(got, "select ?")
self.assertEqual(bound, [r"C:\path\1"])

def _assertSwap(self, tuple):
self.assertEqual(extract_params(tuple[0]), tuple[1])
Expand Down
13 changes: 12 additions & 1 deletion explorer/tests/test_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from datetime import timedelta
import unittest
import os
from urllib.parse import quote
from unittest.mock import Mock, patch, MagicMock
from unittest import skipIf

Expand Down Expand Up @@ -458,7 +459,17 @@ def test_params_in_download(self):
"foo:123"
)
resp = self.client.get(url)
self.assertContains(resp, "'123'")
self.assertContains(resp, "123")

def test_params_in_url_cannot_inject_sql(self):
# Regression: parameter values come from the URL, so they must be bound, not interpolated.
q = SimpleQueryFactory(sql="select '$$who$$' as who;")
url = "{}?params={}".format(
reverse("download_query", kwargs={"query_id": q.id}),
quote("who:') union select 1 where '1'='1", safe="")
)
resp = self.client.get(url)
self.assertNotContains(resp, "django_migrations")

def test_download_defaults_to_csv(self):
query = SimpleQueryFactory()
Expand Down
53 changes: 48 additions & 5 deletions explorer/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,56 @@ def param(name):
return f"{EXPLORER_PARAM_TOKEN}{name}{EXPLORER_PARAM_TOKEN}"


def swap_params(sql, params):
def _param_regex(name, placeholder=None):
token = rf"\$\${re.escape(str(name))}(?:\|([^\$\:]+))?(?:\:([^\$]+))?\$\$"
if placeholder is None:
return re.compile(token, re.I)
# A parameter written as '$$id$$' (quoted, as the docs recommend) has to become a bind
# placeholder *without* its quotes, otherwise the driver sees the literal string "?".
return re.compile(rf"""(['"]){token}\1|{token}""", re.I)


def _bindable(value):
"""
Coerce a parameter value to the type it would have had as literal SQL.

Parameters arrive as strings from the URL. Binding them as strings would make
``WHERE id = $$id$$`` compare a number to text, so numeric-looking values are bound
numerically instead.
"""
if isinstance(value, str):
for converter in (int, float):
try:
return converter(value)
except ValueError:
continue
return value


def swap_params(sql, params, bind=None):
"""
Substitute ``$$name$$`` tokens in ``sql``.

With no ``bind`` argument the values are interpolated straight into the SQL text. That is
only safe for display (logging, the blacklist check). Values come from the request
(``?params=name:value``), so anything that will be *executed* must pass ``bind``.

:param bind: callable taking the number of values bound so far and returning that driver's
bind placeholder. When given, values are collected instead of interpolated.
:return: ``(sql, bound_values)``; ``bound_values`` is empty when ``bind`` is None.
"""
p = params.items() if params else {}
bound = []

def replace(match, value):
if bind is None:
return str(value)
bound.append(_bindable(value))
return bind(len(bound))

for k, v in p:
fmt_k = re.escape(str(k).lower())
regex = re.compile(rf"\$\${fmt_k}(?:\|([^\$\:]+))?(?:\:([^\$]+))?\$\$", re.I)
sql = regex.sub(str(v), sql)
return sql
sql = _param_regex(k, bind).sub(lambda m, v=v: replace(m, v), sql)
return sql, bound


def extract_params(text):
Expand Down
Loading