Skip to content

Commit 11e0b29

Browse files
authored
bpo-43846: Use less stack for large literals and calls (GH-25403)
* Modify compiler to reduce stack consumption for large expressions. * Add more tests for stack usage. * Add NEWS item. * Raise SystemError for truly excessive stack use.
1 parent da74350 commit 11e0b29

File tree

3 files changed

+171
-50
lines changed

3 files changed

+171
-50
lines changed

Lib/test/test_compile.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -961,6 +961,32 @@ def test_if_else(self):
961961
def test_binop(self):
962962
self.check_stack_size("x + " * self.N + "x")
963963

964+
def test_list(self):
965+
self.check_stack_size("[" + "x, " * self.N + "x]")
966+
967+
def test_tuple(self):
968+
self.check_stack_size("(" + "x, " * self.N + "x)")
969+
970+
def test_set(self):
971+
self.check_stack_size("{" + "x, " * self.N + "x}")
972+
973+
def test_dict(self):
974+
self.check_stack_size("{" + "x:x, " * self.N + "x:x}")
975+
976+
def test_func_args(self):
977+
self.check_stack_size("f(" + "x, " * self.N + ")")
978+
979+
def test_func_kwargs(self):
980+
kwargs = (f'a{i}=x' for i in range(self.N))
981+
self.check_stack_size("f(" + ", ".join(kwargs) + ")")
982+
983+
def test_func_args(self):
984+
self.check_stack_size("o.m(" + "x, " * self.N + ")")
985+
986+
def test_meth_kwargs(self):
987+
kwargs = (f'a{i}=x' for i in range(self.N))
988+
self.check_stack_size("o.m(" + ", ".join(kwargs) + ")")
989+
964990
def test_func_and(self):
965991
code = "def f(x):\n"
966992
code += " x and x\n" * self.N
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Data stack usage is much reduced for large literal and call expressions.

Python/compile.c

Lines changed: 144 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,30 @@
4343
#define COMP_SETCOMP 2
4444
#define COMP_DICTCOMP 3
4545

46+
/* A soft limit for stack use, to avoid excessive
47+
* memory use for large constants, etc.
48+
*
49+
* The value 30 is plucked out of thin air.
50+
* Code that could use more stack than this is
51+
* rare, so the exact value is unimportant.
52+
*/
53+
#define STACK_USE_GUIDELINE 30
54+
55+
/* If we exceed this limit, it should
56+
* be considered a compiler bug.
57+
* Currently it should be impossible
58+
* to exceed STACK_USE_GUIDELINE * 100,
59+
* as 100 is the maximum parse depth.
60+
* For performance reasons we will
61+
* want to reduce this to a
62+
* few hundred in the future.
63+
*
64+
* NOTE: Whatever MAX_ALLOWED_STACK_USE is
65+
* set to, it should never restrict what Python
66+
* we can write, just how we compile it.
67+
*/
68+
#define MAX_ALLOWED_STACK_USE (STACK_USE_GUIDELINE * 100)
69+
4670
#define IS_TOP_LEVEL_AWAIT(c) ( \
4771
(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \
4872
&& (c->u->u_ste->ste_type == ModuleBlock))
@@ -1403,7 +1427,7 @@ compiler_addop_name(struct compiler *c, int opcode, PyObject *dict,
14031427
*/
14041428

14051429
static int
1406-
compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
1430+
compiler_addop_i_line(struct compiler *c, int opcode, Py_ssize_t oparg, int lineno)
14071431
{
14081432
struct instr *i;
14091433
int off;
@@ -1424,10 +1448,22 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
14241448
i = &c->u->u_curblock->b_instr[off];
14251449
i->i_opcode = opcode;
14261450
i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
1427-
i->i_lineno = c->u->u_lineno;
1451+
i->i_lineno = lineno;
14281452
return 1;
14291453
}
14301454

1455+
static int
1456+
compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
1457+
{
1458+
return compiler_addop_i_line(c, opcode, oparg, c->u->u_lineno);
1459+
}
1460+
1461+
static int
1462+
compiler_addop_i_noline(struct compiler *c, int opcode, Py_ssize_t oparg)
1463+
{
1464+
return compiler_addop_i_line(c, opcode, oparg, -1);
1465+
}
1466+
14311467
static int add_jump_to_block(basicblock *b, int opcode, int lineno, basicblock *target)
14321468
{
14331469
assert(HAS_ARG(opcode));
@@ -1527,6 +1563,11 @@ compiler_addop_j_noline(struct compiler *c, int opcode, basicblock *b)
15271563
return 0; \
15281564
}
15291565

1566+
#define ADDOP_I_NOLINE(C, OP, O) { \
1567+
if (!compiler_addop_i_noline((C), (OP), (O))) \
1568+
return 0; \
1569+
}
1570+
15301571
#define ADDOP_JUMP(C, OP, O) { \
15311572
if (!compiler_addop_j((C), (OP), (O))) \
15321573
return 0; \
@@ -3707,14 +3748,13 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed,
37073748
int build, int add, int extend, int tuple)
37083749
{
37093750
Py_ssize_t n = asdl_seq_LEN(elts);
3710-
Py_ssize_t i, seen_star = 0;
37113751
if (n > 2 && are_all_items_const(elts, 0, n)) {
37123752
PyObject *folded = PyTuple_New(n);
37133753
if (folded == NULL) {
37143754
return 0;
37153755
}
37163756
PyObject *val;
3717-
for (i = 0; i < n; i++) {
3757+
for (Py_ssize_t i = 0; i < n; i++) {
37183758
val = ((expr_ty)asdl_seq_GET(elts, i))->v.Constant.value;
37193759
Py_INCREF(val);
37203760
PyTuple_SET_ITEM(folded, i, val);
@@ -3735,38 +3775,16 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed,
37353775
return 1;
37363776
}
37373777

3738-
for (i = 0; i < n; i++) {
3778+
int big = n+pushed > STACK_USE_GUIDELINE;
3779+
int seen_star = 0;
3780+
for (Py_ssize_t i = 0; i < n; i++) {
37393781
expr_ty elt = asdl_seq_GET(elts, i);
37403782
if (elt->kind == Starred_kind) {
37413783
seen_star = 1;
37423784
}
37433785
}
3744-
if (seen_star) {
3745-
seen_star = 0;
3746-
for (i = 0; i < n; i++) {
3747-
expr_ty elt = asdl_seq_GET(elts, i);
3748-
if (elt->kind == Starred_kind) {
3749-
if (seen_star == 0) {
3750-
ADDOP_I(c, build, i+pushed);
3751-
seen_star = 1;
3752-
}
3753-
VISIT(c, expr, elt->v.Starred.value);
3754-
ADDOP_I(c, extend, 1);
3755-
}
3756-
else {
3757-
VISIT(c, expr, elt);
3758-
if (seen_star) {
3759-
ADDOP_I(c, add, 1);
3760-
}
3761-
}
3762-
}
3763-
assert(seen_star);
3764-
if (tuple) {
3765-
ADDOP(c, LIST_TO_TUPLE);
3766-
}
3767-
}
3768-
else {
3769-
for (i = 0; i < n; i++) {
3786+
if (!seen_star && !big) {
3787+
for (Py_ssize_t i = 0; i < n; i++) {
37703788
expr_ty elt = asdl_seq_GET(elts, i);
37713789
VISIT(c, expr, elt);
37723790
}
@@ -3775,6 +3793,33 @@ starunpack_helper(struct compiler *c, asdl_expr_seq *elts, int pushed,
37753793
} else {
37763794
ADDOP_I(c, build, n+pushed);
37773795
}
3796+
return 1;
3797+
}
3798+
int sequence_built = 0;
3799+
if (big) {
3800+
ADDOP_I(c, build, pushed);
3801+
sequence_built = 1;
3802+
}
3803+
for (Py_ssize_t i = 0; i < n; i++) {
3804+
expr_ty elt = asdl_seq_GET(elts, i);
3805+
if (elt->kind == Starred_kind) {
3806+
if (sequence_built == 0) {
3807+
ADDOP_I(c, build, i+pushed);
3808+
sequence_built = 1;
3809+
}
3810+
VISIT(c, expr, elt->v.Starred.value);
3811+
ADDOP_I(c, extend, 1);
3812+
}
3813+
else {
3814+
VISIT(c, expr, elt);
3815+
if (sequence_built) {
3816+
ADDOP_I(c, add, 1);
3817+
}
3818+
}
3819+
}
3820+
assert(sequence_built);
3821+
if (tuple) {
3822+
ADDOP(c, LIST_TO_TUPLE);
37783823
}
37793824
return 1;
37803825
}
@@ -3874,7 +3919,8 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end
38743919
{
38753920
Py_ssize_t i, n = end - begin;
38763921
PyObject *keys, *key;
3877-
if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) {
3922+
int big = n*2 > STACK_USE_GUIDELINE;
3923+
if (n > 1 && !big && are_all_items_const(e->v.Dict.keys, begin, end)) {
38783924
for (i = begin; i < end; i++) {
38793925
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
38803926
}
@@ -3889,12 +3935,19 @@ compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end
38893935
}
38903936
ADDOP_LOAD_CONST_NEW(c, keys);
38913937
ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
3938+
return 1;
38923939
}
3893-
else {
3894-
for (i = begin; i < end; i++) {
3895-
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
3896-
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3940+
if (big) {
3941+
ADDOP_I(c, BUILD_MAP, 0);
3942+
}
3943+
for (i = begin; i < end; i++) {
3944+
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
3945+
VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3946+
if (big) {
3947+
ADDOP_I(c, MAP_ADD, 1);
38973948
}
3949+
}
3950+
if (!big) {
38983951
ADDOP_I(c, BUILD_MAP, n);
38993952
}
39003953
return 1;
@@ -3930,7 +3983,7 @@ compiler_dict(struct compiler *c, expr_ty e)
39303983
ADDOP_I(c, DICT_UPDATE, 1);
39313984
}
39323985
else {
3933-
if (elements == 0xFFFF) {
3986+
if (elements*2 > STACK_USE_GUIDELINE) {
39343987
if (!compiler_subdict(c, e, i - elements, i + 1)) {
39353988
return 0;
39363989
}
@@ -4126,11 +4179,15 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e)
41264179
/* Check that the call node is an attribute access, and that
41274180
the call doesn't have keyword parameters. */
41284181
if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load ||
4129-
asdl_seq_LEN(e->v.Call.keywords))
4182+
asdl_seq_LEN(e->v.Call.keywords)) {
41304183
return -1;
4131-
4132-
/* Check that there are no *varargs types of arguments. */
4184+
}
4185+
/* Check that there aren't too many arguments */
41334186
argsl = asdl_seq_LEN(args);
4187+
if (argsl >= STACK_USE_GUIDELINE) {
4188+
return -1;
4189+
}
4190+
/* Check that there are no *varargs types of arguments. */
41344191
for (i = 0; i < argsl; i++) {
41354192
expr_ty elt = asdl_seq_GET(args, i);
41364193
if (elt->kind == Starred_kind) {
@@ -4192,9 +4249,29 @@ compiler_call(struct compiler *c, expr_ty e)
41924249
static int
41934250
compiler_joined_str(struct compiler *c, expr_ty e)
41944251
{
4195-
VISIT_SEQ(c, expr, e->v.JoinedStr.values);
4196-
if (asdl_seq_LEN(e->v.JoinedStr.values) != 1)
4197-
ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values));
4252+
4253+
Py_ssize_t value_count = asdl_seq_LEN(e->v.JoinedStr.values);
4254+
if (value_count > STACK_USE_GUIDELINE) {
4255+
ADDOP_LOAD_CONST_NEW(c, _PyUnicode_FromASCII("", 0));
4256+
PyObject *join = _PyUnicode_FromASCII("join", 4);
4257+
if (join == NULL) {
4258+
return 0;
4259+
}
4260+
ADDOP_NAME(c, LOAD_METHOD, join, names);
4261+
Py_DECREF(join);
4262+
ADDOP_I(c, BUILD_LIST, 0);
4263+
for (Py_ssize_t i = 0; i < asdl_seq_LEN(e->v.JoinedStr.values); i++) {
4264+
VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i));
4265+
ADDOP_I(c, LIST_APPEND, 1);
4266+
}
4267+
ADDOP_I(c, CALL_METHOD, 1);
4268+
}
4269+
else {
4270+
VISIT_SEQ(c, expr, e->v.JoinedStr.values);
4271+
if (asdl_seq_LEN(e->v.JoinedStr.values) != 1) {
4272+
ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values));
4273+
}
4274+
}
41984275
return 1;
41994276
}
42004277

@@ -4251,7 +4328,8 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be
42514328
keyword_ty kw;
42524329
PyObject *keys, *key;
42534330
assert(n > 0);
4254-
if (n > 1) {
4331+
int big = n*2 > STACK_USE_GUIDELINE;
4332+
if (n > 1 && !big) {
42554333
for (i = begin; i < end; i++) {
42564334
kw = asdl_seq_GET(keywords, i);
42574335
VISIT(c, expr, kw->value);
@@ -4267,14 +4345,20 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be
42674345
}
42684346
ADDOP_LOAD_CONST_NEW(c, keys);
42694347
ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
4348+
return 1;
42704349
}
4271-
else {
4272-
/* a for loop only executes once */
4273-
for (i = begin; i < end; i++) {
4274-
kw = asdl_seq_GET(keywords, i);
4275-
ADDOP_LOAD_CONST(c, kw->arg);
4276-
VISIT(c, expr, kw->value);
4350+
if (big) {
4351+
ADDOP_I_NOLINE(c, BUILD_MAP, 0);
4352+
}
4353+
for (i = begin; i < end; i++) {
4354+
kw = asdl_seq_GET(keywords, i);
4355+
ADDOP_LOAD_CONST(c, kw->arg);
4356+
VISIT(c, expr, kw->value);
4357+
if (big) {
4358+
ADDOP_I_NOLINE(c, MAP_ADD, 1);
42774359
}
4360+
}
4361+
if (!big) {
42784362
ADDOP_I(c, BUILD_MAP, n);
42794363
}
42804364
return 1;
@@ -4296,6 +4380,9 @@ compiler_call_helper(struct compiler *c,
42964380
nelts = asdl_seq_LEN(args);
42974381
nkwelts = asdl_seq_LEN(keywords);
42984382

4383+
if (nelts + nkwelts*2 > STACK_USE_GUIDELINE) {
4384+
goto ex_call;
4385+
}
42994386
for (i = 0; i < nelts; i++) {
43004387
expr_ty elt = asdl_seq_GET(args, i);
43014388
if (elt->kind == Starred_kind) {
@@ -6603,6 +6690,13 @@ makecode(struct compiler *c, struct assembler *a, PyObject *consts)
66036690
Py_DECREF(consts);
66046691
goto error;
66056692
}
6693+
if (maxdepth > MAX_ALLOWED_STACK_USE) {
6694+
PyErr_Format(PyExc_SystemError,
6695+
"excessive stack use: stack is %d deep",
6696+
maxdepth);
6697+
Py_DECREF(consts);
6698+
goto error;
6699+
}
66066700
co = PyCode_NewWithPosOnlyArgs(posonlyargcount+posorkeywordargcount,
66076701
posonlyargcount, kwonlyargcount, nlocals_int,
66086702
maxdepth, flags, a->a_bytecode, consts, names,

0 commit comments

Comments
 (0)