diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 89b077ac428345..77c55cda474a97 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2219,17 +2219,17 @@ dummy_func( } inst(JUMP_FORWARD, (--)) { - JUMPBY(oparg); + JUMP_POP_DISPATCH(oparg, 0); } inst(JUMP_BACKWARD, (--)) { CHECK_EVAL_BREAKER(); + #if ENABLE_SPECIALIZATION _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); - JUMPBY(1-oparg); - #if ENABLE_SPECIALIZATION here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); if (here[1].cache > tstate->interp->optimizer_backedge_threshold) { + JUMPBY(1 - oparg); OBJECT_STAT_INC(optimization_attempts); frame = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (frame == NULL) { @@ -2241,6 +2241,7 @@ dummy_func( goto resume_frame; } #endif /* ENABLE_SPECIALIZATION */ + JUMP_POP_DISPATCH(1 - oparg, 0); } pseudo(JUMP) = { @@ -2272,24 +2273,28 @@ dummy_func( inst(POP_JUMP_IF_FALSE, (cond -- )) { assert(PyBool_Check(cond)); - JUMPBY(oparg * Py_IsFalse(cond)); + if (Py_IsFalse(cond)) { + JUMP_POP_DISPATCH(oparg, 1); + } } inst(POP_JUMP_IF_TRUE, (cond -- )) { assert(PyBool_Check(cond)); - JUMPBY(oparg * Py_IsTrue(cond)); + if (Py_IsTrue(cond)) { + JUMP_POP_DISPATCH(oparg, 1); + } } inst(POP_JUMP_IF_NOT_NONE, (value -- )) { if (!Py_IsNone(value)) { DECREF_INPUTS(); - JUMPBY(oparg); + JUMP_POP_DISPATCH(oparg, 1); } } inst(POP_JUMP_IF_NONE, (value -- )) { if (Py_IsNone(value)) { - JUMPBY(oparg); + JUMP_POP_DISPATCH(oparg, 1); } else { DECREF_INPUTS(); @@ -2302,7 +2307,7 @@ dummy_func( * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ - JUMPBY(-oparg); + JUMP_POP_DISPATCH(-oparg, 0); } inst(GET_LEN, (obj -- obj, len_o)) { @@ -2410,18 +2415,17 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, frame->prev_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ + #if ENABLE_SPECIALIZATION assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + #endif Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); } // Common case: no jump, leave it to the code generator } @@ -2468,11 +2472,8 @@ dummy_func( Py_DECREF(seq); } Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); end_for_iter_list: // Common case: no jump, leave it to the code generator } @@ -2491,11 +2492,8 @@ dummy_func( Py_DECREF(seq); } Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); end_for_iter_tuple: // Common case: no jump, leave it to the code generator } @@ -2505,12 +2503,9 @@ dummy_func( DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { - STACK_SHRINK(1); + // STACK_SHRINK(1); Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); } long value = r->start; r->start = value + r->step; diff --git a/Python/ceval.c b/Python/ceval.c index 0ee95bc3a3a4bb..66c015b1b46803 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2715,6 +2715,10 @@ void Py_LeaveRecursiveCall(void) ///////////////////// Experimental UOp Interpreter ///////////////////// +#undef JUMP_POP_DISPATCH +#define JUMP_POP_DISPATCH(x, n) \ + do { frame->prev_instr += (x); stack_pointer -= (n); goto exit; } while (0) + #undef DEOPT_IF #define DEOPT_IF(COND, INSTNAME) \ if ((COND)) { \ @@ -2772,6 +2776,13 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject #define ENABLE_SPECIALIZATION 0 #include "executor_cases.c.h" + case JUMP_TO_TOP: + { + pc = 0; + CHECK_EVAL_BREAKER(); + break; + } + case SAVE_IP: { frame->prev_instr = ip_offset + oparg; @@ -2795,6 +2806,12 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject } } +exit: + DPRINTF(2, "Jumping!\n"); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(self); + return frame; + unbound_local_error: format_exc_check_arg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 72800aaaaa2ac4..aa161349026490 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -155,6 +155,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define JUMPBY(x) (next_instr += (x)) #define SKIP_OVER(x) (next_instr += (x)) +#define JUMP_POP_DISPATCH(x, n) do { JUMPBY(x); STACK_SHRINK(n); DISPATCH(); } while (0) + /* OpCode prediction macros Some opcodes tend to come in pairs thus making it possible to predict the second code when the first is run. For example, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 29ebb0b3e8b2d9..55ad235d25b71e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1626,16 +1626,89 @@ break; } + case JUMP_FORWARD: { + #line 2222 "Python/bytecodes.c" + JUMP_POP_DISPATCH(oparg, 0); + #line 1633 "Python/executor_cases.c.h" + break; + } + + case POP_JUMP_IF_FALSE: { + PyObject *cond = stack_pointer[-1]; + #line 2275 "Python/bytecodes.c" + assert(PyBool_Check(cond)); + if (Py_IsFalse(cond)) { + JUMP_POP_DISPATCH(oparg, 1); + } + #line 1644 "Python/executor_cases.c.h" + STACK_SHRINK(1); + break; + } + + case POP_JUMP_IF_TRUE: { + PyObject *cond = stack_pointer[-1]; + #line 2282 "Python/bytecodes.c" + assert(PyBool_Check(cond)); + if (Py_IsTrue(cond)) { + JUMP_POP_DISPATCH(oparg, 1); + } + #line 1656 "Python/executor_cases.c.h" + STACK_SHRINK(1); + break; + } + + case POP_JUMP_IF_NOT_NONE: { + PyObject *value = stack_pointer[-1]; + #line 2289 "Python/bytecodes.c" + if (!Py_IsNone(value)) { + #line 1665 "Python/executor_cases.c.h" + Py_DECREF(value); + #line 2291 "Python/bytecodes.c" + JUMP_POP_DISPATCH(oparg, 1); + } + #line 1670 "Python/executor_cases.c.h" + STACK_SHRINK(1); + break; + } + + case POP_JUMP_IF_NONE: { + PyObject *value = stack_pointer[-1]; + #line 2296 "Python/bytecodes.c" + if (Py_IsNone(value)) { + JUMP_POP_DISPATCH(oparg, 1); + } + else { + #line 1682 "Python/executor_cases.c.h" + Py_DECREF(value); + #line 2301 "Python/bytecodes.c" + } + #line 1686 "Python/executor_cases.c.h" + STACK_SHRINK(1); + break; + } + + case JUMP_BACKWARD_NO_INTERRUPT: { + #line 2305 "Python/bytecodes.c" + /* This bytecode is used in the `yield from` or `await` loop. + * If there is an interrupt, we want it handled in the innermost + * generator or coroutine, so we deliberately do not check it here. + * (see bpo-30039). + */ + JUMP_POP_DISPATCH(-oparg, 0); + #line 1699 "Python/executor_cases.c.h" + break; + } + case GET_LEN: { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2309 "Python/bytecodes.c" + #line 2314 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 1639 "Python/executor_cases.c.h" + #line 1712 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; break; @@ -1646,16 +1719,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2317 "Python/bytecodes.c" + #line 2322 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 1655 "Python/executor_cases.c.h" + #line 1728 "Python/executor_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2322 "Python/bytecodes.c" + #line 2327 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -1663,7 +1736,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_None; // Failure! } - #line 1667 "Python/executor_cases.c.h" + #line 1740 "Python/executor_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; break; @@ -1672,10 +1745,10 @@ case MATCH_MAPPING: { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2332 "Python/bytecodes.c" + #line 2337 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 1679 "Python/executor_cases.c.h" + #line 1752 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -1684,10 +1757,10 @@ case MATCH_SEQUENCE: { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2337 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 1691 "Python/executor_cases.c.h" + #line 1764 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -1697,11 +1770,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2342 "Python/bytecodes.c" + #line 2347 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 1705 "Python/executor_cases.c.h" + #line 1778 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; break; @@ -1710,14 +1783,14 @@ case GET_ITER: { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2348 "Python/bytecodes.c" + #line 2353 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 1717 "Python/executor_cases.c.h" + #line 1790 "Python/executor_cases.c.h" Py_DECREF(iterable); - #line 2351 "Python/bytecodes.c" + #line 2356 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 1721 "Python/executor_cases.c.h" + #line 1794 "Python/executor_cases.c.h" stack_pointer[-1] = iter; break; } @@ -1725,7 +1798,7 @@ case GET_YIELD_FROM_ITER: { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2355 "Python/bytecodes.c" + #line 2360 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -1748,21 +1821,100 @@ if (iter == NULL) { goto error; } - #line 1752 "Python/executor_cases.c.h" + #line 1825 "Python/executor_cases.c.h" Py_DECREF(iterable); - #line 2378 "Python/bytecodes.c" + #line 2383 "Python/bytecodes.c" } - #line 1756 "Python/executor_cases.c.h" + #line 1829 "Python/executor_cases.c.h" stack_pointer[-1] = iter; break; } + case FOR_ITER_LIST: { + PyObject *iter = stack_pointer[-1]; + PyObject *next; + #line 2462 "Python/bytecodes.c" + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + _PyListIterObject *it = (_PyListIterObject *)iter; + STAT_INC(FOR_ITER, hit); + PyListObject *seq = it->it_seq; + if (seq) { + if (it->it_index < PyList_GET_SIZE(seq)) { + next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + goto end_for_iter_list; // End of this instruction + } + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); + end_for_iter_list: + // Common case: no jump, leave it to the code generator + #line 1855 "Python/executor_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + break; + } + + case FOR_ITER_TUPLE: { + PyObject *iter = stack_pointer[-1]; + PyObject *next; + #line 2482 "Python/bytecodes.c" + _PyTupleIterObject *it = (_PyTupleIterObject *)iter; + DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); + PyTupleObject *seq = it->it_seq; + if (seq) { + if (it->it_index < PyTuple_GET_SIZE(seq)) { + next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + goto end_for_iter_tuple; // End of this instruction + } + it->it_seq = NULL; + Py_DECREF(seq); + } + Py_DECREF(iter); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); + end_for_iter_tuple: + // Common case: no jump, leave it to the code generator + #line 1882 "Python/executor_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + break; + } + + case FOR_ITER_RANGE: { + PyObject *iter = stack_pointer[-1]; + PyObject *next; + #line 2502 "Python/bytecodes.c" + _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + // STACK_SHRINK(1); + Py_DECREF(r); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); + } + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) { + goto error; + } + #line 1907 "Python/executor_cases.c.h" + STACK_GROW(1); + stack_pointer[-1] = next; + break; + } + case WITH_EXCEPT_START: { PyObject *val = stack_pointer[-1]; PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2610 "Python/bytecodes.c" + #line 2605 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -1783,7 +1935,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 1787 "Python/executor_cases.c.h" + #line 1939 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; break; @@ -1792,7 +1944,7 @@ case PUSH_EXC_INFO: { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2649 "Python/bytecodes.c" + #line 2644 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -1802,7 +1954,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 1806 "Python/executor_cases.c.h" + #line 1958 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -1811,7 +1963,7 @@ case EXIT_INIT_CHECK: { PyObject *should_be_none = stack_pointer[-1]; - #line 3019 "Python/bytecodes.c" + #line 3014 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -1819,7 +1971,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 1823 "Python/executor_cases.c.h" + #line 1975 "Python/executor_cases.c.h" STACK_SHRINK(1); break; } @@ -1827,7 +1979,7 @@ case MAKE_FUNCTION: { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3433 "Python/bytecodes.c" + #line 3428 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -1839,7 +1991,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 1843 "Python/executor_cases.c.h" + #line 1995 "Python/executor_cases.c.h" stack_pointer[-1] = func; break; } @@ -1847,7 +1999,7 @@ case SET_FUNCTION_ATTRIBUTE: { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3447 "Python/bytecodes.c" + #line 3442 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -1872,7 +2024,7 @@ default: Py_UNREACHABLE(); } - #line 1876 "Python/executor_cases.c.h" + #line 2028 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; break; @@ -1883,15 +2035,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3497 "Python/bytecodes.c" + #line 3492 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 1889 "Python/executor_cases.c.h" + #line 2041 "Python/executor_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3499 "Python/bytecodes.c" + #line 3494 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 1895 "Python/executor_cases.c.h" + #line 2047 "Python/executor_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -1901,14 +2053,14 @@ case CONVERT_VALUE: { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3503 "Python/bytecodes.c" + #line 3498 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 1912 "Python/executor_cases.c.h" + #line 2064 "Python/executor_cases.c.h" stack_pointer[-1] = result; break; } @@ -1916,7 +2068,7 @@ case FORMAT_SIMPLE: { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3512 "Python/bytecodes.c" + #line 3507 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -1927,7 +2079,7 @@ else { res = value; } - #line 1931 "Python/executor_cases.c.h" + #line 2083 "Python/executor_cases.c.h" stack_pointer[-1] = res; break; } @@ -1936,12 +2088,12 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3525 "Python/bytecodes.c" + #line 3520 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 1945 "Python/executor_cases.c.h" + #line 2097 "Python/executor_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; break; @@ -1950,10 +2102,10 @@ case COPY: { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3532 "Python/bytecodes.c" + #line 3527 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 1957 "Python/executor_cases.c.h" + #line 2109 "Python/executor_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; break; @@ -1962,9 +2114,9 @@ case SWAP: { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3557 "Python/bytecodes.c" + #line 3552 "Python/bytecodes.c" assert(oparg >= 2); - #line 1968 "Python/executor_cases.c.h" + #line 2120 "Python/executor_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eb8b50e5c905d4..8dae355d77f38c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3175,20 +3175,19 @@ TARGET(JUMP_FORWARD) { #line 2222 "Python/bytecodes.c" - JUMPBY(oparg); + JUMP_POP_DISPATCH(oparg, 0); #line 3180 "Python/generated_cases.c.h" - DISPATCH(); } TARGET(JUMP_BACKWARD) { #line 2226 "Python/bytecodes.c" CHECK_EVAL_BREAKER(); + #if ENABLE_SPECIALIZATION _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); - JUMPBY(1-oparg); - #if ENABLE_SPECIALIZATION here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); if (here[1].cache > tstate->interp->optimizer_backedge_threshold) { + JUMPBY(1 - oparg); OBJECT_STAT_INC(optimization_attempts); frame = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (frame == NULL) { @@ -3200,12 +3199,12 @@ goto resume_frame; } #endif /* ENABLE_SPECIALIZATION */ + JUMP_POP_DISPATCH(1 - oparg, 0); #line 3204 "Python/generated_cases.c.h" - DISPATCH(); } TARGET(ENTER_EXECUTOR) { - #line 2257 "Python/bytecodes.c" + #line 2258 "Python/bytecodes.c" CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); @@ -3220,81 +3219,84 @@ goto resume_with_error; } goto resume_frame; - #line 3224 "Python/generated_cases.c.h" + #line 3223 "Python/generated_cases.c.h" } TARGET(POP_JUMP_IF_FALSE) { PyObject *cond = stack_pointer[-1]; - #line 2274 "Python/bytecodes.c" + #line 2275 "Python/bytecodes.c" assert(PyBool_Check(cond)); - JUMPBY(oparg * Py_IsFalse(cond)); - #line 3232 "Python/generated_cases.c.h" + if (Py_IsFalse(cond)) { + JUMP_POP_DISPATCH(oparg, 1); + } + #line 3233 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2279 "Python/bytecodes.c" + #line 2282 "Python/bytecodes.c" assert(PyBool_Check(cond)); - JUMPBY(oparg * Py_IsTrue(cond)); - #line 3242 "Python/generated_cases.c.h" + if (Py_IsTrue(cond)) { + JUMP_POP_DISPATCH(oparg, 1); + } + #line 3245 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2284 "Python/bytecodes.c" + #line 2289 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 3251 "Python/generated_cases.c.h" + #line 3254 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2286 "Python/bytecodes.c" - JUMPBY(oparg); + #line 2291 "Python/bytecodes.c" + JUMP_POP_DISPATCH(oparg, 1); } - #line 3256 "Python/generated_cases.c.h" + #line 3259 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2291 "Python/bytecodes.c" + #line 2296 "Python/bytecodes.c" if (Py_IsNone(value)) { - JUMPBY(oparg); + JUMP_POP_DISPATCH(oparg, 1); } else { - #line 3268 "Python/generated_cases.c.h" + #line 3271 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2296 "Python/bytecodes.c" + #line 2301 "Python/bytecodes.c" } - #line 3272 "Python/generated_cases.c.h" + #line 3275 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2300 "Python/bytecodes.c" + #line 2305 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ - JUMPBY(-oparg); - #line 3285 "Python/generated_cases.c.h" - DISPATCH(); + JUMP_POP_DISPATCH(-oparg, 0); + #line 3288 "Python/generated_cases.c.h" } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2309 "Python/bytecodes.c" + #line 2314 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3298 "Python/generated_cases.c.h" + #line 3300 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3305,16 +3307,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2317 "Python/bytecodes.c" + #line 2322 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3314 "Python/generated_cases.c.h" + #line 3316 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2322 "Python/bytecodes.c" + #line 2327 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3322,7 +3324,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_None; // Failure! } - #line 3326 "Python/generated_cases.c.h" + #line 3328 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3331,10 +3333,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2332 "Python/bytecodes.c" + #line 2337 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 3338 "Python/generated_cases.c.h" + #line 3340 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3343,10 +3345,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2337 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 3350 "Python/generated_cases.c.h" + #line 3352 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3356,11 +3358,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2342 "Python/bytecodes.c" + #line 2347 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3364 "Python/generated_cases.c.h" + #line 3366 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3369,14 +3371,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2348 "Python/bytecodes.c" + #line 2353 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3376 "Python/generated_cases.c.h" + #line 3378 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2351 "Python/bytecodes.c" + #line 2356 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3380 "Python/generated_cases.c.h" + #line 3382 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3384,7 +3386,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2355 "Python/bytecodes.c" + #line 2360 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3407,11 +3409,11 @@ if (iter == NULL) { goto error; } - #line 3411 "Python/generated_cases.c.h" + #line 3413 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2378 "Python/bytecodes.c" + #line 2383 "Python/bytecodes.c" } - #line 3415 "Python/generated_cases.c.h" + #line 3417 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3421,7 +3423,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2396 "Python/bytecodes.c" + #line 2401 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3439,21 +3441,20 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, frame->prev_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ + #if ENABLE_SPECIALIZATION assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + #endif Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); } // Common case: no jump, leave it to the code generator - #line 3457 "Python/generated_cases.c.h" + #line 3458 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3461,7 +3462,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2430 "Python/bytecodes.c" + #line 2434 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3487,14 +3488,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3491 "Python/generated_cases.c.h" + #line 3492 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2458 "Python/bytecodes.c" + #line 2462 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3508,14 +3509,11 @@ Py_DECREF(seq); } Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3519 "Python/generated_cases.c.h" + #line 3517 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3525,7 +3523,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2481 "Python/bytecodes.c" + #line 2482 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3539,14 +3537,11 @@ Py_DECREF(seq); } Py_DECREF(iter); - STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3550 "Python/generated_cases.c.h" + #line 3545 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3556,17 +3551,14 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2504 "Python/bytecodes.c" + #line 2502 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { - STACK_SHRINK(1); + // STACK_SHRINK(1); Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); - // Jump over END_FOR instruction. - JUMPBY(oparg + 1); - DISPATCH(); + JUMP_POP_DISPATCH(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1, 1); } long value = r->start; r->start = value + r->step; @@ -3575,7 +3567,7 @@ if (next == NULL) { goto error; } - #line 3579 "Python/generated_cases.c.h" + #line 3571 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3584,7 +3576,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2525 "Python/bytecodes.c" + #line 2520 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3600,14 +3592,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3604 "Python/generated_cases.c.h" + #line 3596 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2543 "Python/bytecodes.c" + #line 2538 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3630,16 +3622,16 @@ Py_DECREF(enter); goto error; } - #line 3634 "Python/generated_cases.c.h" + #line 3626 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2566 "Python/bytecodes.c" + #line 2561 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3643 "Python/generated_cases.c.h" + #line 3635 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3650,7 +3642,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2575 "Python/bytecodes.c" + #line 2570 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3676,16 +3668,16 @@ Py_DECREF(enter); goto error; } - #line 3680 "Python/generated_cases.c.h" + #line 3672 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2601 "Python/bytecodes.c" + #line 2596 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3689 "Python/generated_cases.c.h" + #line 3681 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3697,7 +3689,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2610 "Python/bytecodes.c" + #line 2605 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3718,7 +3710,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3722 "Python/generated_cases.c.h" + #line 3714 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3727,7 +3719,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2649 "Python/bytecodes.c" + #line 2644 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3737,7 +3729,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3741 "Python/generated_cases.c.h" + #line 3733 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3751,7 +3743,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2661 "Python/bytecodes.c" + #line 2656 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3768,7 +3760,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3772 "Python/generated_cases.c.h" + #line 3764 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3782,7 +3774,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2680 "Python/bytecodes.c" + #line 2675 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3792,7 +3784,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3796 "Python/generated_cases.c.h" + #line 3788 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3806,7 +3798,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2692 "Python/bytecodes.c" + #line 2687 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3820,7 +3812,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3824 "Python/generated_cases.c.h" + #line 3816 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3829,16 +3821,16 @@ } TARGET(KW_NAMES) { - #line 2708 "Python/bytecodes.c" + #line 2703 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(FRAME_CO_CONSTS)); kwnames = GETITEM(FRAME_CO_CONSTS, oparg); - #line 3837 "Python/generated_cases.c.h" + #line 3829 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2714 "Python/bytecodes.c" + #line 2709 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3851,7 +3843,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3855 "Python/generated_cases.c.h" + #line 3847 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3861,7 +3853,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2760 "Python/bytecodes.c" + #line 2755 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3943,7 +3935,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3947 "Python/generated_cases.c.h" + #line 3939 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3955,7 +3947,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2848 "Python/bytecodes.c" + #line 2843 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3965,7 +3957,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3969 "Python/generated_cases.c.h" + #line 3961 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3974,7 +3966,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2860 "Python/bytecodes.c" + #line 2855 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -4000,7 +3992,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 4004 "Python/generated_cases.c.h" + #line 3996 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -4008,7 +4000,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2888 "Python/bytecodes.c" + #line 2883 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -4044,7 +4036,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 4048 "Python/generated_cases.c.h" + #line 4040 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -4052,7 +4044,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2926 "Python/bytecodes.c" + #line 2921 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4062,7 +4054,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 4066 "Python/generated_cases.c.h" + #line 4058 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4075,7 +4067,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2938 "Python/bytecodes.c" + #line 2933 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4086,7 +4078,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4090 "Python/generated_cases.c.h" + #line 4082 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4100,7 +4092,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2952 "Python/bytecodes.c" + #line 2947 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -4111,7 +4103,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4115 "Python/generated_cases.c.h" + #line 4107 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4124,7 +4116,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; - #line 2966 "Python/bytecodes.c" + #line 2961 "Python/bytecodes.c" /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -4175,12 +4167,12 @@ * as it will be checked after start_frame */ tstate->py_recursion_remaining--; goto start_frame; - #line 4179 "Python/generated_cases.c.h" + #line 4171 "Python/generated_cases.c.h" } TARGET(EXIT_INIT_CHECK) { PyObject *should_be_none = stack_pointer[-1]; - #line 3019 "Python/bytecodes.c" + #line 3014 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -4188,7 +4180,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 4192 "Python/generated_cases.c.h" + #line 4184 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -4198,7 +4190,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3029 "Python/bytecodes.c" + #line 3024 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4220,7 +4212,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4224 "Python/generated_cases.c.h" + #line 4216 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4234,7 +4226,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3054 "Python/bytecodes.c" + #line 3049 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4262,7 +4254,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4266 "Python/generated_cases.c.h" + #line 4258 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4276,7 +4268,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3085 "Python/bytecodes.c" + #line 3080 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4308,7 +4300,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4312 "Python/generated_cases.c.h" + #line 4304 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4322,7 +4314,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3120 "Python/bytecodes.c" + #line 3115 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4354,7 +4346,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4358 "Python/generated_cases.c.h" + #line 4350 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4368,7 +4360,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3155 "Python/bytecodes.c" + #line 3150 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4393,7 +4385,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4397 "Python/generated_cases.c.h" + #line 4389 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4406,7 +4398,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3182 "Python/bytecodes.c" + #line 3177 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4433,7 +4425,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4437 "Python/generated_cases.c.h" + #line 4429 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4445,7 +4437,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3212 "Python/bytecodes.c" + #line 3207 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4463,14 +4455,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4467 "Python/generated_cases.c.h" + #line 4459 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3232 "Python/bytecodes.c" + #line 3227 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4501,7 +4493,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4505 "Python/generated_cases.c.h" + #line 4497 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4514,7 +4506,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3266 "Python/bytecodes.c" + #line 3261 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4543,7 +4535,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4547 "Python/generated_cases.c.h" + #line 4539 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4556,7 +4548,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3298 "Python/bytecodes.c" + #line 3293 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4585,7 +4577,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4589 "Python/generated_cases.c.h" + #line 4581 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4598,7 +4590,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3330 "Python/bytecodes.c" + #line 3325 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4626,7 +4618,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4630 "Python/generated_cases.c.h" + #line 4622 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4636,9 +4628,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3361 "Python/bytecodes.c" + #line 3356 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4642 "Python/generated_cases.c.h" + #line 4634 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4647,7 +4639,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3365 "Python/bytecodes.c" + #line 3360 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4709,14 +4701,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4713 "Python/generated_cases.c.h" + #line 4705 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3427 "Python/bytecodes.c" + #line 3422 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4720 "Python/generated_cases.c.h" + #line 4712 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4727,7 +4719,7 @@ TARGET(MAKE_FUNCTION) { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3433 "Python/bytecodes.c" + #line 3428 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4739,7 +4731,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4743 "Python/generated_cases.c.h" + #line 4735 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4747,7 +4739,7 @@ TARGET(SET_FUNCTION_ATTRIBUTE) { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3447 "Python/bytecodes.c" + #line 3442 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4772,14 +4764,14 @@ default: Py_UNREACHABLE(); } - #line 4776 "Python/generated_cases.c.h" + #line 4768 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3474 "Python/bytecodes.c" + #line 3469 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4800,7 +4792,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4804 "Python/generated_cases.c.h" + #line 4796 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4808,15 +4800,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3497 "Python/bytecodes.c" + #line 3492 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4814 "Python/generated_cases.c.h" + #line 4806 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3499 "Python/bytecodes.c" + #line 3494 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4820 "Python/generated_cases.c.h" + #line 4812 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4826,14 +4818,14 @@ TARGET(CONVERT_VALUE) { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3503 "Python/bytecodes.c" + #line 3498 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4837 "Python/generated_cases.c.h" + #line 4829 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4841,7 +4833,7 @@ TARGET(FORMAT_SIMPLE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3512 "Python/bytecodes.c" + #line 3507 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4852,7 +4844,7 @@ else { res = value; } - #line 4856 "Python/generated_cases.c.h" + #line 4848 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4861,12 +4853,12 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3525 "Python/bytecodes.c" + #line 3520 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4870 "Python/generated_cases.c.h" + #line 4862 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4875,10 +4867,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3532 "Python/bytecodes.c" + #line 3527 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4882 "Python/generated_cases.c.h" + #line 4874 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4890,7 +4882,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3537 "Python/bytecodes.c" + #line 3532 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4905,12 +4897,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4909 "Python/generated_cases.c.h" + #line 4901 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3552 "Python/bytecodes.c" + #line 3547 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4914 "Python/generated_cases.c.h" + #line 4906 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4920,16 +4912,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3557 "Python/bytecodes.c" + #line 3552 "Python/bytecodes.c" assert(oparg >= 2); - #line 4926 "Python/generated_cases.c.h" + #line 4918 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3561 "Python/bytecodes.c" + #line 3556 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4941,48 +4933,48 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4945 "Python/generated_cases.c.h" + #line 4937 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3575 "Python/bytecodes.c" + #line 3570 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4951 "Python/generated_cases.c.h" + #line 4943 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3579 "Python/bytecodes.c" + #line 3574 "Python/bytecodes.c" CHECK_EVAL_BREAKER(); INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4959 "Python/generated_cases.c.h" + #line 4951 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3584 "Python/bytecodes.c" + #line 3579 "Python/bytecodes.c" PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; int offset = Py_IsTrue(cond) * oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4970 "Python/generated_cases.c.h" + #line 4962 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3592 "Python/bytecodes.c" + #line 3587 "Python/bytecodes.c" PyObject *cond = POP(); assert(PyBool_Check(cond)); _Py_CODEUNIT *here = next_instr - 1; int offset = Py_IsFalse(cond) * oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4981 "Python/generated_cases.c.h" + #line 4973 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3600 "Python/bytecodes.c" + #line 3595 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4994,12 +4986,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4998 "Python/generated_cases.c.h" + #line 4990 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3614 "Python/bytecodes.c" + #line 3609 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -5011,30 +5003,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 5015 "Python/generated_cases.c.h" + #line 5007 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3628 "Python/bytecodes.c" + #line 3623 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 5026 "Python/generated_cases.c.h" + #line 5018 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3636 "Python/bytecodes.c" + #line 3631 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 5033 "Python/generated_cases.c.h" + #line 5025 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3641 "Python/bytecodes.c" + #line 3636 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 5040 "Python/generated_cases.c.h" + #line 5032 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index ac86a4abd9c1b3..7d091b855e8c81 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -21,18 +21,19 @@ #define EXIT_TRACE 300 #define SAVE_IP 301 -#define _GUARD_BOTH_INT 302 -#define _BINARY_OP_MULTIPLY_INT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_SUBTRACT_INT 305 -#define _GUARD_BOTH_FLOAT 306 -#define _BINARY_OP_MULTIPLY_FLOAT 307 -#define _BINARY_OP_ADD_FLOAT 308 -#define _BINARY_OP_SUBTRACT_FLOAT 309 -#define _GUARD_BOTH_UNICODE 310 -#define _BINARY_OP_ADD_UNICODE 311 -#define _LOAD_LOCALS 312 -#define _LOAD_FROM_DICT_OR_GLOBALS 313 +#define JUMP_TO_TOP 302 +#define _GUARD_BOTH_INT 303 +#define _BINARY_OP_MULTIPLY_INT 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_SUBTRACT_INT 306 +#define _GUARD_BOTH_FLOAT 307 +#define _BINARY_OP_MULTIPLY_FLOAT 308 +#define _BINARY_OP_ADD_FLOAT 309 +#define _BINARY_OP_SUBTRACT_FLOAT 310 +#define _GUARD_BOTH_UNICODE 311 +#define _BINARY_OP_ADD_UNICODE 312 +#define _LOAD_LOCALS 313 +#define _LOAD_FROM_DICT_OR_GLOBALS 314 #ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -1244,6 +1245,12 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [CONTAINS_OP] = { .nuops = 1, .uops = { { CONTAINS_OP, 0, 0 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { CHECK_EG_MATCH, 0, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { CHECK_EXC_MATCH, 0, 0 } } }, + [JUMP_FORWARD] = { .nuops = 1, .uops = { { JUMP_FORWARD, 0, 0 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { POP_JUMP_IF_FALSE, 0, 0 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { POP_JUMP_IF_TRUE, 0, 0 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 1, .uops = { { POP_JUMP_IF_NOT_NONE, 0, 0 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 1, .uops = { { POP_JUMP_IF_NONE, 0, 0 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { JUMP_BACKWARD_NO_INTERRUPT, 0, 0 } } }, [GET_LEN] = { .nuops = 1, .uops = { { GET_LEN, 0, 0 } } }, [MATCH_CLASS] = { .nuops = 1, .uops = { { MATCH_CLASS, 0, 0 } } }, [MATCH_MAPPING] = { .nuops = 1, .uops = { { MATCH_MAPPING, 0, 0 } } }, @@ -1251,6 +1258,9 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [MATCH_KEYS] = { .nuops = 1, .uops = { { MATCH_KEYS, 0, 0 } } }, [GET_ITER] = { .nuops = 1, .uops = { { GET_ITER, 0, 0 } } }, [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, + [FOR_ITER_LIST] = { .nuops = 1, .uops = { { FOR_ITER_LIST, 0, 0 } } }, + [FOR_ITER_TUPLE] = { .nuops = 1, .uops = { { FOR_ITER_TUPLE, 0, 0 } } }, + [FOR_ITER_RANGE] = { .nuops = 1, .uops = { { FOR_ITER_RANGE, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { EXIT_INIT_CHECK, 0, 0 } } }, @@ -1267,18 +1277,19 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { const char * const _PyOpcode_uop_name[512] = { [300] = "EXIT_TRACE", [301] = "SAVE_IP", - [302] = "_GUARD_BOTH_INT", - [303] = "_BINARY_OP_MULTIPLY_INT", - [304] = "_BINARY_OP_ADD_INT", - [305] = "_BINARY_OP_SUBTRACT_INT", - [306] = "_GUARD_BOTH_FLOAT", - [307] = "_BINARY_OP_MULTIPLY_FLOAT", - [308] = "_BINARY_OP_ADD_FLOAT", - [309] = "_BINARY_OP_SUBTRACT_FLOAT", - [310] = "_GUARD_BOTH_UNICODE", - [311] = "_BINARY_OP_ADD_UNICODE", - [312] = "_LOAD_LOCALS", - [313] = "_LOAD_FROM_DICT_OR_GLOBALS", + [302] = "JUMP_TO_TOP", + [303] = "_GUARD_BOTH_INT", + [304] = "_BINARY_OP_MULTIPLY_INT", + [305] = "_BINARY_OP_ADD_INT", + [306] = "_BINARY_OP_SUBTRACT_INT", + [307] = "_GUARD_BOTH_FLOAT", + [308] = "_BINARY_OP_MULTIPLY_FLOAT", + [309] = "_BINARY_OP_ADD_FLOAT", + [310] = "_BINARY_OP_SUBTRACT_FLOAT", + [311] = "_GUARD_BOTH_UNICODE", + [312] = "_BINARY_OP_ADD_UNICODE", + [313] = "_LOAD_LOCALS", + [314] = "_LOAD_FROM_DICT_OR_GLOBALS", }; #endif // NEED_OPCODE_METADATA #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index db117bb180c1c8..00524c8be0d81b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -374,9 +374,7 @@ translate_bytecode_to_trace( _PyUOpInstruction *trace, int max_length) { -#ifdef Py_DEBUG _Py_CODEUNIT *initial_instr = instr; -#endif int trace_length = 0; #ifdef Py_DEBUG @@ -419,6 +417,12 @@ translate_bytecode_to_trace( opcode = instr->op.code; operand = (operand << 8) | instr->op.arg; } + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[operand&255]; + opcode = executor->vm_data.opcode; + DPRINTF(2, " * ENTER_EXECUTOR -> %s\n", _PyOpcode_OpName[opcode]); + operand = (operand & 0xffffff00) | executor->vm_data.oparg; + } switch (opcode) { case LOAD_FAST_LOAD_FAST: case STORE_FAST_LOAD_FAST: @@ -449,6 +453,18 @@ translate_bytecode_to_trace( } break; } + case JUMP_BACKWARD: + { + if (instr + 2 - operand == initial_instr + && trace_length + 3 <= max_length) + { + ADD_TO_TRACE(JUMP_TO_TOP, 0); + } + else { + DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); + } + goto done; + } default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index a90abfe20c1739..876b2d80345b85 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -264,7 +264,10 @@ def fromInstruction(instr: "AnyInstruction"): HAS_ARG_FLAG=variable_used(instr, "oparg"), HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), - HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), + HAS_JUMP_FLAG=( + variable_used(instr, "JUMPBY") + or variable_used(instr, "JUMP_POP_DISPATCH") + ), ) @staticmethod @@ -314,12 +317,18 @@ class ActiveCacheEffect: "oparg1", # Proxy for super-instructions like LOAD_FAST_LOAD_FAST "JUMPBY", "DISPATCH", + "GO_TO_INSTRUCTION", + "SKIP_OVER", "INSTRUMENTED_JUMP", "throwflag", "exception_unwind", "import_from", "import_name", "_PyObject_CallNoArgs", # Proxy for BEFORE_WITH + "cframe", + "resume_frame", + "CACHE", + "RESERVED", ) @@ -399,9 +408,6 @@ def __init__(self, inst: parser.InstDef): def is_viable_uop(self) -> bool: """Whether this instruction is viable as a uop.""" - if self.always_exits: - # print(f"Skipping {self.name} because it always exits") - return False if self.instr_flags.HAS_ARG_FLAG: # If the instruction uses oparg, it cannot use any caches if self.active_caches: @@ -1326,6 +1332,7 @@ def add(name: str) -> None: counter += 1 add("EXIT_TRACE") add("SAVE_IP") + add("JUMP_TO_TOP") for instr in self.instrs.values(): if instr.kind == "op" and instr.is_viable_uop(): add(instr.name) @@ -1573,6 +1580,7 @@ def always_exits(lines: list[str]) -> bool: "goto ", "return ", "DISPATCH", + "JUMP_POP_DISPATCH", "GO_TO_", "Py_UNREACHABLE()", "ERROR_IF(true, ",