diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index a9db8860e91c06c..a6fb8ddbda97cc7 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -39,6 +39,13 @@ extern PyObject* _PyObject_Call_Prepend( PyObject *args, PyObject *kwargs); +PyAPI_FUNC(int) _PyObject_CallSetItemDunder( + PyThreadState *tstate, + PyObject *func, + PyObject *self, + PyObject *key, + PyObject *value); + extern PyObject* _PyObject_VectorcallDictTstate( PyThreadState *tstate, PyObject *callable, diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 293c1ea4414e23e..1b46c7676a8a30c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -131,6 +131,7 @@ typedef struct { typedef struct { _Py_BackoffCounter counter; + uint16_t version[2]; } _PyStoreSubscrCache; #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index b6945f2bc5f6e0d..b7efe112e171e9e 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -303,6 +303,7 @@ Known values: Python 3.16a1 3703 (Replace DELETE_GLOBAL with PUSH_NULL; STORE_GLOBAL) Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) + Python 3.16a1 3706 (Add STORE_SUBSCR_PY_DUNDER) Python 3.17 will start with 3750 @@ -312,7 +313,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3705 +#define PYC_MAGIC_NUMBER 3706 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 38bccdd48c0f862..f4a391b8f744320 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -482,6 +482,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 3; case STORE_SUBSCR_LIST_INT: return 3; + case STORE_SUBSCR_PY_DUNDER: + return 3; case SWAP: return 2 + (oparg-2); case TO_BOOL: @@ -977,6 +979,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case STORE_SUBSCR_LIST_INT: return 0; + case STORE_SUBSCR_PY_DUNDER: + return 0; case SWAP: return 2 + (oparg-2); case TO_BOOL: @@ -1309,9 +1313,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_PY_DUNDER] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1524,9 +1529,10 @@ _PyOpcode_macro_expansion[256] = { [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, OPARG_SIMPLE, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 4, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_STORE_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 2 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 4, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_STORE_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 3 }, { _POP_TOP_INT, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [STORE_SUBSCR_PY_DUNDER] = { .nuops = 1, .uops = { { _STORE_SUBSCR_PY_DUNDER, 2, 1 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, OPARG_SIMPLE, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, OPARG_SIMPLE, 2 } } }, [TO_BOOL_ALWAYS_TRUE] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, @@ -1774,6 +1780,7 @@ const char *_PyOpcode_OpName[267] = { [STORE_SUBSCR] = "STORE_SUBSCR", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [STORE_SUBSCR_PY_DUNDER] = "STORE_SUBSCR_PY_DUNDER", [SWAP] = "SWAP", [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", @@ -1801,7 +1808,7 @@ PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; const uint8_t _PyOpcode_Caches[256] = { [RESUME] = 1, [TO_BOOL] = 3, - [STORE_SUBSCR] = 1, + [STORE_SUBSCR] = 3, [SEND] = 1, [FOR_ITER] = 1, [UNPACK_SEQUENCE] = 1, @@ -1838,7 +1845,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [219] = 219, [220] = 220, [221] = 221, [222] = 222, @@ -2064,6 +2070,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_SUBSCR] = STORE_SUBSCR, [STORE_SUBSCR_DICT] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [STORE_SUBSCR_PY_DUNDER] = STORE_SUBSCR, [SWAP] = SWAP, [TO_BOOL] = TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, @@ -2099,7 +2106,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 219: \ case 220: \ case 221: \ case 222: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 625c33a6640b853..f372a9c6547943d 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -336,6 +336,7 @@ enum { _STORE_SUBSCR_DICT, _STORE_SUBSCR_DICT_KNOWN_HASH, _STORE_SUBSCR_LIST_INT, + _STORE_SUBSCR_PY_DUNDER, _SWAP, _SWAP_2, _SWAP_3, @@ -365,7 +366,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 653 +#define MAX_UOP_ID 654 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -1348,6 +1349,7 @@ enum { _STORE_SUBSCR_DICT_r31, _STORE_SUBSCR_DICT_KNOWN_HASH_r31, _STORE_SUBSCR_LIST_INT_r32, + _STORE_SUBSCR_PY_DUNDER_r30, _SWAP_r11, _SWAP_2_r02, _SWAP_2_r12, @@ -1439,7 +1441,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1650 +#define MAX_UOP_REGS_ID 1652 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f464effe23bc0b3..2210671592e4874 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -157,6 +157,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_PY_DUNDER] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1543,6 +1544,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 0, 3, _STORE_SUBSCR_r30 }, }, }, + [_STORE_SUBSCR_PY_DUNDER] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SUBSCR_PY_DUNDER_r30 }, + }, + }, [_STORE_SUBSCR_LIST_INT] = { .best = { 3, 3, 3, 3 }, .entries = { @@ -4251,6 +4261,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LIST_APPEND_r10] = _LIST_APPEND, [_SET_ADD_r10] = _SET_ADD, [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_STORE_SUBSCR_PY_DUNDER_r30] = _STORE_SUBSCR_PY_DUNDER, [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, @@ -6040,6 +6051,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = "_STORE_SUBSCR_DICT_KNOWN_HASH_r31", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_STORE_SUBSCR_LIST_INT_r32] = "_STORE_SUBSCR_LIST_INT_r32", + [_STORE_SUBSCR_PY_DUNDER] = "_STORE_SUBSCR_PY_DUNDER", + [_STORE_SUBSCR_PY_DUNDER_r30] = "_STORE_SUBSCR_PY_DUNDER_r30", [_SWAP] = "_SWAP", [_SWAP_r11] = "_SWAP_r11", [_SWAP_2] = "_SWAP_2", @@ -6410,6 +6423,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_SUBSCR: return 3; + case _STORE_SUBSCR_PY_DUNDER: + return 3; case _STORE_SUBSCR_LIST_INT: return 3; case _STORE_SUBSCR_DICT: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6c..b8d5c25a2268844 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -209,15 +209,16 @@ extern "C" { #define STORE_ATTR_WITH_HINT 207 #define STORE_SUBSCR_DICT 208 #define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define STORE_SUBSCR_PY_DUNDER 210 +#define TO_BOOL_ALWAYS_TRUE 211 +#define TO_BOOL_BOOL 212 +#define TO_BOOL_INT 213 +#define TO_BOOL_LIST 214 +#define TO_BOOL_NONE 215 +#define TO_BOOL_STR 216 +#define UNPACK_SEQUENCE_LIST 217 +#define UNPACK_SEQUENCE_TUPLE 218 +#define UNPACK_SEQUENCE_TWO_TUPLE 219 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d2488..ef5eb45b5b4d8cc 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -36,6 +36,7 @@ STORE_SUBSCR=( "STORE_SUBSCR_DICT", "STORE_SUBSCR_LIST_INT", + "STORE_SUBSCR_PY_DUNDER", ), SEND=( "SEND_GEN", @@ -215,15 +216,16 @@ STORE_ATTR_WITH_HINT=207, STORE_SUBSCR_DICT=208, STORE_SUBSCR_LIST_INT=209, - TO_BOOL_ALWAYS_TRUE=210, - TO_BOOL_BOOL=211, - TO_BOOL_INT=212, - TO_BOOL_LIST=213, - TO_BOOL_NONE=214, - TO_BOOL_STR=215, - UNPACK_SEQUENCE_LIST=216, - UNPACK_SEQUENCE_TUPLE=217, - UNPACK_SEQUENCE_TWO_TUPLE=218, + STORE_SUBSCR_PY_DUNDER=210, + TO_BOOL_ALWAYS_TRUE=211, + TO_BOOL_BOOL=212, + TO_BOOL_INT=213, + TO_BOOL_LIST=214, + TO_BOOL_NONE=215, + TO_BOOL_STR=216, + UNPACK_SEQUENCE_LIST=217, + UNPACK_SEQUENCE_TUPLE=218, + UNPACK_SEQUENCE_TWO_TUPLE=219, ) opmap = frozendict( diff --git a/Lib/opcode.py b/Lib/opcode.py index 750d83b2c87af58..33f7039e92e3bc0 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -102,6 +102,7 @@ ), STORE_SUBSCR=frozendict( counter=1, + version=2, ), SEND=frozendict( counter=1, diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7946550ec0db637..8d6dee34862c964 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -691,6 +691,36 @@ def write(items): opname = "BINARY_OP_SUBSCR_GETITEM" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization + def test_store_subscr_py_dunder(self): + def get_items(): + class C: + __setitem__ = lambda self, item, value: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item[None] = None + except TypeError: + pass + + def write(items): + for item in items: + try: + del item.__setitem__ + except AttributeError: + pass + type(item).__setitem__ = lambda self, item, value: None + + opname = "STORE_SUBSCR_PY_DUNDER" + self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_binary_subscr_list_int(self): def get_items(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-18-08-55-00.gh-issue-143732.Xk3mPq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-18-08-55-00.gh-issue-143732.Xk3mPq.rst new file mode 100644 index 000000000000000..e36e686b341b68c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-18-08-55-00.gh-issue-143732.Xk3mPq.rst @@ -0,0 +1,4 @@ +Specialize :opcode:`STORE_SUBSCR` for container objects whose type defines +``__setitem__`` as a simple Python function. The new ``STORE_SUBSCR_PY_DUNDER`` +specialization guards on the container's type version and calls the dunder +directly, skipping the generic ``PyObject_SetItem`` slot dispatch. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 7a75e80298fcd82..300ccbfb53ede48 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -12602,10 +12602,10 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 4; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef container; _PyStackRef sub; @@ -12629,6 +12629,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 2 cache entries */ // _STORE_SUBSCR { v = stack_pointer[-3]; @@ -12674,9 +12675,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 4; INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 3, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; _PyStackRef dict_st; @@ -12697,7 +12698,7 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } - /* Skip 1 cache entry */ + /* Skip 3 cache entries */ // _STORE_SUBSCR_DICT { sub = stack_pointer[-1]; @@ -12744,9 +12745,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 4; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef list_st; @@ -12773,7 +12774,7 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } - /* Skip 1 cache entry */ + /* Skip 3 cache entries */ // _STORE_SUBSCR_LIST_INT { sub_st = value; @@ -12837,6 +12838,77 @@ DISPATCH(); } + TARGET(STORE_SUBSCR_PY_DUNDER) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_PY_DUNDER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(STORE_SUBSCR_PY_DUNDER); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 3, "incorrect cache size"); + _PyStackRef v; + _PyStackRef container; + _PyStackRef sub; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; + uint32_t version = read_u32(&this_instr[2].cache); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyTypeObject *tp = Py_TYPE(container_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != version) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *setitem = _PyType_Lookup(tp, &_Py_ID(__setitem__)); + _PyFrame_StackPointerInvalidate(frame); + if (setitem == NULL || !PyFunction_Check(setitem)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _PyObject_CallSetItemDunder(tstate, setitem, container_o, + PyStackRef_AsPyObjectBorrow(sub), + PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + TARGET(SWAP) { #if _Py_TAIL_CALL_INTERP int opcode = SWAP; diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..19276e861cf4cf7 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -210,6 +210,7 @@ static void *opcode_targets_table[256] = { &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_SUBSCR_PY_DUNDER, &&TARGET_TO_BOOL_ALWAYS_TRUE, &&TARGET_TO_BOOL_BOOL, &&TARGET_TO_BOOL_INT, @@ -232,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -478,7 +478,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -743,6 +743,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_PY_DUNDER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -986,6 +987,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_STORE_SUBSCR_PY_DUNDER, [SWAP] = _TAIL_CALL_SWAP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, @@ -1016,7 +1018,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1244,6 +1245,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_TRACE_RECORD, [SWAP] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, @@ -1274,7 +1276,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Objects/call.c b/Objects/call.c index 9718642473103cf..44343d1e933ffd0 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -396,6 +396,20 @@ PyObject_CallOneArg(PyObject *func, PyObject *arg) } +int +_PyObject_CallSetItemDunder(PyThreadState *tstate, PyObject *func, + PyObject *self, PyObject *key, PyObject *value) +{ + PyObject *args[3] = {self, key, value}; + PyObject *res = _PyObject_VectorcallTstate(tstate, func, args, 3, NULL); + if (res == NULL) { + return -1; + } + Py_DECREF(res); + return 0; +} + + /* --- PyFunction call functions ---------------------------------- */ PyObject * diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fb0cdf4d65e060d..25a704699df80c8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1393,6 +1393,7 @@ dummy_func( family(STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = { STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT, + STORE_SUBSCR_PY_DUNDER, }; specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { @@ -1414,10 +1415,27 @@ dummy_func( ERROR_IF(err); } - macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + unused/2 + _STORE_SUBSCR; macro(STORE_SUBSCR_LIST_INT) = - _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; + _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/3 + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; + + op(_STORE_SUBSCR_PY_DUNDER, (version/2, v, container, sub -- )) { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyTypeObject *tp = Py_TYPE(container_o); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != version); + PyObject *setitem = _PyType_Lookup(tp, &_Py_ID(__setitem__)); + DEOPT_IF(setitem == NULL || !PyFunction_Check(setitem)); + STAT_INC(STORE_SUBSCR, hit); + int err = _PyObject_CallSetItemDunder(tstate, setitem, container_o, + PyStackRef_AsPyObjectBorrow(sub), + PyStackRef_AsPyObjectBorrow(v)); + DECREF_INPUTS(); + ERROR_IF(err); + } + + macro(STORE_SUBSCR_PY_DUNDER) = + unused/1 + _STORE_SUBSCR_PY_DUNDER; op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); @@ -1452,7 +1470,7 @@ dummy_func( macro(STORE_SUBSCR_DICT) = _RECORD_NOS_TYPE + - _GUARD_NOS_DICT_STORE_SUBSCRIPT + unused/1 + _STORE_SUBSCR_DICT + POP_TOP; + _GUARD_NOS_DICT_STORE_SUBSCRIPT + unused/3 + _STORE_SUBSCR_DICT + POP_TOP; op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- st)) { PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9aad9e003765cf8..aca41fc14e6a4e8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8426,6 +8426,90 @@ break; } + case _STORE_SUBSCR_PY_DUNDER_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + container = _stack_item_1; + v = _stack_item_0; + uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyTypeObject *tp = Py_TYPE(container_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != version) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub; + _tos_cache1 = container; + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + stack_pointer[0] = v; + stack_pointer[1] = container; + stack_pointer[2] = sub; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *setitem = _PyType_Lookup(tp, &_Py_ID(__setitem__)); + _PyFrame_StackPointerInvalidate(frame); + if (setitem == NULL || !PyFunction_Check(setitem)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub; + _tos_cache1 = container; + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(STORE_SUBSCR, hit); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _PyObject_CallSetItemDunder(tstate, setitem, container_o, + PyStackRef_AsPyObjectBorrow(sub), + PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _STORE_SUBSCR_LIST_INT_r32: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 77c18b3d61fefc7..4977fee4ee9ccbe 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12599,10 +12599,10 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 4; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; + _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef container; _PyStackRef sub; @@ -12626,6 +12626,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } + /* Skip 2 cache entries */ // _STORE_SUBSCR { v = stack_pointer[-3]; @@ -12671,9 +12672,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 4; INSTRUCTION_STATS(STORE_SUBSCR_DICT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 3, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; _PyStackRef dict_st; @@ -12694,7 +12695,7 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } - /* Skip 1 cache entry */ + /* Skip 3 cache entries */ // _STORE_SUBSCR_DICT { sub = stack_pointer[-1]; @@ -12741,9 +12742,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 2; + next_instr += 4; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef list_st; @@ -12770,7 +12771,7 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } - /* Skip 1 cache entry */ + /* Skip 3 cache entries */ // _STORE_SUBSCR_LIST_INT { sub_st = value; @@ -12834,6 +12835,77 @@ DISPATCH(); } + TARGET(STORE_SUBSCR_PY_DUNDER) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_PY_DUNDER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(STORE_SUBSCR_PY_DUNDER); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 3, "incorrect cache size"); + _PyStackRef v; + _PyStackRef container; + _PyStackRef sub; + /* Skip 1 cache entry */ + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; + uint32_t version = read_u32(&this_instr[2].cache); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyTypeObject *tp = Py_TYPE(container_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != version) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *setitem = _PyType_Lookup(tp, &_Py_ID(__setitem__)); + _PyFrame_StackPointerInvalidate(frame); + if (setitem == NULL || !PyFunction_Check(setitem)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _PyObject_CallSetItemDunder(tstate, setitem, container_o, + PyStackRef_AsPyObjectBorrow(sub), + PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + TARGET(SWAP) { #if _Py_TAIL_CALL_INTERP int opcode = SWAP; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..19276e861cf4cf7 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -210,6 +210,7 @@ static void *opcode_targets_table[256] = { &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_SUBSCR_PY_DUNDER, &&TARGET_TO_BOOL_ALWAYS_TRUE, &&TARGET_TO_BOOL_BOOL, &&TARGET_TO_BOOL_INT, @@ -232,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -478,7 +478,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -743,6 +743,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_PY_DUNDER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -986,6 +987,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_STORE_SUBSCR_PY_DUNDER, [SWAP] = _TAIL_CALL_SWAP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, @@ -1016,7 +1018,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1244,6 +1245,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_PY_DUNDER] = _TAIL_CALL_TRACE_RECORD, [SWAP] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, @@ -1274,7 +1276,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, [222] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 21f275f27cafe04..0612ed13da7356f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1729,6 +1729,13 @@ break; } + case _STORE_SUBSCR_PY_DUNDER: { + CHECK_STACK_BOUNDS(-3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _STORE_SUBSCR_LIST_INT: { JitOptRef sub_st; JitOptRef list_st; diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e202..bc4fefad502dcee 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -134,6 +134,7 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [BINARY_OP_SUBSCR_GETITEM] = {2, {_RECORD_NOS_INDEX, _RECORD_TOS_TYPE_INDEX}}, [STORE_SUBSCR] = {1, {_RECORD_NOS_TYPE_INDEX}}, [STORE_SUBSCR_LIST_INT] = {1, {_RECORD_NOS_TYPE_INDEX}}, + [STORE_SUBSCR_PY_DUNDER] = {1, {_RECORD_NOS_TYPE_INDEX}}, [STORE_SUBSCR_DICT] = {1, {_RECORD_NOS_TYPE_INDEX}}, [SEND] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, [SEND_GEN] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, diff --git a/Python/specialize.c b/Python/specialize.c index 05cb76ff015ff40..26aa826e01fbc2c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1619,6 +1619,26 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD specialize(instr, STORE_SUBSCR_DICT); return; } + { + unsigned int tp_version; + PyObject *descr = _PyType_LookupRefAndVersion(container_type, + &_Py_ID(__setitem__), &tp_version); + if (descr != NULL && Py_IS_TYPE(descr, &PyFunction_Type) && tp_version != 0) { + PyCodeObject *fcode = (PyCodeObject *)((PyFunctionObject *)descr)->func_code; + if ((fcode->co_flags & (CO_VARKEYWORDS | CO_VARARGS)) == 0 && + fcode->co_kwonlyargcount == 0 && + (fcode->co_flags & CO_OPTIMIZED) && + fcode->co_argcount == 3) + { + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); + write_u32(cache->version, tp_version); + specialize(instr, STORE_SUBSCR_PY_DUNDER); + Py_DECREF(descr); + return; + } + } + Py_XDECREF(descr); + } SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub)); unspecialize(instr); }