From 85cc1050e63692ef6bec20751687d21fc738dd38 Mon Sep 17 00:00:00 2001 From: Francois Chagnon Date: Tue, 13 Oct 2026 15:35:44 -0400 Subject: [PATCH 1/3] Recognize modern loop back edges at EXTENDED_ARG prefixes --- src/decompiler.rs | 16 ++- tests/fixtures/extended_loop_prefix.py | 123 ++++++++++++++++++++++++ tests/integration.rs | 14 +++ tests/pyc/extended_loop_prefix.3.11.pyc | Bin 0 -> 4315 bytes tests/pyc/extended_loop_prefix.3.12.pyc | Bin 0 -> 3100 bytes tests/pyc/extended_loop_prefix.3.13.pyc | Bin 0 -> 3107 bytes tests/pyc/extended_loop_prefix.3.14.pyc | Bin 0 -> 3115 bytes 7 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/extended_loop_prefix.py create mode 100644 tests/pyc/extended_loop_prefix.3.11.pyc create mode 100644 tests/pyc/extended_loop_prefix.3.12.pyc create mode 100644 tests/pyc/extended_loop_prefix.3.13.pyc create mode 100644 tests/pyc/extended_loop_prefix.3.14.pyc diff --git a/src/decompiler.rs b/src/decompiler.rs index 33a3465d..9e22e2f9 100644 --- a/src/decompiler.rs +++ b/src/decompiler.rs @@ -23194,14 +23194,20 @@ impl<'a> Ctx<'a> { // so offsets inside a cached instruction have no idx_of entry // either - a back edge landing <=8 bytes before the loop top // would falsely match a CACHE tail (compileall 3.13 main tore - // into an INCOMPLETE stub). The EXTENDED_ARG-prefix shape this - // recognizes is pre-3.11 wordcode only (no CACHE gaps). - if self.version.at_least(3, 11) { - return false; - } + // into an INCOMPLETE stub). Modern offsets therefore need proof from + // original bytes, rather than absence from the instruction map. if t >= start || start - t > 8 { return false; } + if self.version.at_least(3, 11) { + if t % 2 != 0 || start % 2 != 0 { + return false; + } + if !(t..start).step_by(2).all(|offset| self.code.code.get(offset) + .map_or(false, |byte| self.table.op(*byte) == Op::EXTENDED_ARG)) { + return false; + } + } // no instruction STARTS at t (idx_of also maps END offsets: the // preceding instruction's end lands on t and must not count as // an instruction start) diff --git a/tests/fixtures/extended_loop_prefix.py b/tests/fixtures/extended_loop_prefix.py new file mode 100644 index 00000000..706a8340 --- /dev/null +++ b/tests/fixtures/extended_loop_prefix.py @@ -0,0 +1,123 @@ +# Versioned .pyc files are consumed by tests/integration.rs. + +# The long functions require EXTENDED_ARG before FOR_ITER. + +def size_1(rows, work): + result = [] + for row in rows: + try: + work(row) + except ValueError: + continue + result.append(row) + return result + + +def size_30(rows, work): + result = [] + for row in rows: + try: + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + except ValueError: + continue + result.append(row) + return result + + +def size_60(rows, work): + result = [] + for row in rows: + try: + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + except ValueError: + continue + result.append(row) + return result diff --git a/tests/integration.rs b/tests/integration.rs index f4af5082..985d2196 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -169,3 +169,17 @@ fn py314_hello() { "py3.14 hello", ); } + +#[test] +fn extended_loop_prefix_preserves_handler_continue() { + for version in ["3.11", "3.12", "3.13", "3.14"] { + let (src, clean) = decompile_fixture(&format!("extended_loop_prefix.{version}.pyc")); + assert!(clean, "Python {version}: output marked incomplete:\n{src}"); + for count in [1, 30, 60] { + let body = " work(row)\n".repeat(count); + let expected = format!( + "def size_{count}(rows, work):\n result = []\n for row in rows:\n try:\n{body} except ValueError:\n continue\n result.append(row)"); + assert_contains(&src, &[&expected], version); + } + } +} diff --git a/tests/pyc/extended_loop_prefix.3.11.pyc b/tests/pyc/extended_loop_prefix.3.11.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c805c5c27817f7cd0ae7bda0a15f9b440c6f1a46 GIT binary patch literal 4315 zcmeHK&1(}u6n`_DO_Ob6l4dK~NNMT^6vejqfuI!8MohpW6oiT)lr!h+@QoiqNEjcevpTR%UDAYTSrd@YCK39Bcyd4u6M+Pp({T3TSJ$+aEmp9{V5p zYfJAuBkw&UZ>?`*FjT&#`S3xNL76_2qT(8fioG-N8S}CVoeb1SGT%Q3y))xjG0^yi zI&t{7ama*rDS(SBK#dr;eqRG{Re`Gj<2uARt=RyAO>oBPV-8sCJvGGwI12LnTc|PJ z^c@!?p8$c=ckBoH?%=XBsKoaw2K{ui`V-wm0sX3*Lzw1jOp3{LLzZIW;zpDp_Eygt zN_v-i(&Q^gJu!S%a}?2u!_GaNQz7{q;qaovi@p_|8tZ>h(W&wJS1bD2DHg(CaSks{ z#T`I*MZZXRN$1FG`ixW4`AKB2D!22T|2x)yzq3lniBY;aDkV;co5u;_+=O7Aj;%em zMaw$0xI|c6@NLh9ds!u)R%e)uHh#j_E+OCnn2@IAAwx~DVgFG+wBb6rOt_&pvM{>& E0=Nl$S^xk5 literal 0 HcmV?d00001 diff --git a/tests/pyc/extended_loop_prefix.3.12.pyc b/tests/pyc/extended_loop_prefix.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65423109bbea1f59350bfc4706c8d8a2cf227084 GIT binary patch literal 3100 zcmeHJ&ubGw6n<}ZH_65R!EiSQ3TOlW(@^+QeWLPYIdBn_q8+nK$3}z4@qVK?wl- zAJ(>BJq)gVrM6eF(W>`TV-wQw5}rUB$zqz3WvkM#yvRI%0EOCOyZW-b;u);7A|zpc z5MW(Wfk6?A()-4~2t~Fmyj#jqkMen$ydJ|vusp9zz$Jt^C|}7#L7Ia+R&l1v?BMJ| zCZ|twZgAVTkj?5V>8fnY+{l}@oHzJ$+sAb?m$7U~jLEcNoXY-?rJI&H@^q}$zkl6vY( zC{l@)?>RnD!*8_>t+MzjFyNdZ2PXufXz9V;cRxf1YJkGG&ylF!H$$}FMqr4&Q%998 z997zfcANsf35TJr;5?nBU2t22-Rn1luhZgjJXtDO4z#6j@sFUPR2FA9Wogw$j%2Xd9&n zy_rZPBWydOCeJC`gGjbLdE*rGCi4F1krx1e%M6`H&fa-DTU`J5{r*av_x>QxtnB|6 zalVn@CrNg03UU6{BF^J@pw466sLtJxVVdvIl5cJ^@qW%7c1*=nTY1M0M^fBH?W8w>Lif=uAJrQfc07h&5moZv@gcFES H9;2zB<@P%r literal 0 HcmV?d00001 diff --git a/tests/pyc/extended_loop_prefix.3.13.pyc b/tests/pyc/extended_loop_prefix.3.13.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1f773a3e4d022977cb881eda63696015b07d5ff GIT binary patch literal 3107 zcmeH}&u5E3f-r}-^W)8A_wD!d-q?yFO91fl z&6m&XwO#%Z`%WX>o;`(~EyzO@7~~0BlBa0tLuuBvOdWp!H}=JGHtXz2K#dkagX~#= znsg2dP@wmOcV9c|k&6KYKzLq&X?WmxdZFY3oR2{+2~dHXQVc4jMV4BW z%lAvwhQ7#{!MLYXuj{pP%f(&HST(t8)nJdfhv{acYH>--;W-e;#xGgAX_<-2%9_U+D7->uR{AC95s14r?eI=*d zuSv>%a&)s8^DX!j-3-ZkPKj@SyvO5S{S8|@2J(_2Isd7|pjZVl=yT?f872s10s`hm z>m!q9(Tmv)%++tj&S=Hxm_dz+%)MYch4bL#o#=f zPcEo*Ta_24-P>`3_GEr%Fx)--ejoEIw0W#P3>%;0_Z{y1`oI70Z}Ricf%%!{0{@nu zQ%N>~>F2u0&!3Lv=ebeL&!bRddcUEs;@qO5;@qYKDo!SzrF4hN*$MYfn4n#nl{z=m zPQ~uXq6Ibz0}cwh@5W=J+^FhvEP`{W6)?r^9KInA5(XY8&_C}QCB?|=vXLCS-6347iaF{oI7*ReBXEO)iYkV002Ms z9?yMDzq-ALeXEgXuX+wUB}l_=bW2Qi(8Sq0Q z1fU7$Aw$L?BV2Fp9ELPq;ofa6eZ(Uf25`Xz$j}Lhz$433gdDzW48jnC)2j4`kNfIlm7u>m@kMZ=58=&KYjc_+Z?Mz+TJ8>5Pe!r0Yb^AW9mXL9elY ztYMsmjMIQQ&{%|E9E_%;j`nLY=EISnuLqF9O}Kw+z&&EYy@~pMcpPEwXJl@iaOx&V zJ9l?2?#h^)Hw&>vnQqIT#f$dsae}rpgRfw^TNwOnz+gZ6yiUnn!1#ynw$Mq33cm*Y7-|Zyw{>UXF N*H*{^Au Date: Tue, 13 Oct 2026 18:33:07 -0400 Subject: [PATCH 2/3] Add harness coverage for the reconstruction fix --- .../behavior/cases/b_extended_loop_prefix.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/behavior/cases/b_extended_loop_prefix.py diff --git a/tests/behavior/cases/b_extended_loop_prefix.py b/tests/behavior/cases/b_extended_loop_prefix.py new file mode 100644 index 00000000..f8203427 --- /dev/null +++ b/tests/behavior/cases/b_extended_loop_prefix.py @@ -0,0 +1,50 @@ +# MIN_VERSION: 3.11 + + +def size_30(rows, work): + result = [] + for row in rows: + try: + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + work(row) + except ValueError: + continue + result.append(row) + return result + + +calls = [] +def work(row): + calls.append(row) + if row == 2: + raise ValueError(row) +print(size_30([1, 2, 3], work)) +print(calls) From ae75257b3662c57606993905d70f1ef889bea3ea Mon Sep 17 00:00:00 2001 From: Francois Chagnon Date: Tue, 13 Oct 2026 18:47:52 -0400 Subject: [PATCH 3/3] Preserve prefixed loop back edges in conditional branches --- src/decompiler.rs | 31 +++++++-- .../cases/b_conditional_loop_prefix.py | 63 ++++++++++++++++++ tests/fixtures/conditional_loop_prefix.py | 20 ++++++ tests/integration.rs | 28 ++++++++ tests/pyc/conditional_loop_prefix.3.11.pyc | Bin 0 -> 949 bytes tests/pyc/conditional_loop_prefix.3.12.pyc | Bin 0 -> 1224 bytes tests/pyc/conditional_loop_prefix.3.13.pyc | Bin 0 -> 1276 bytes tests/pyc/conditional_loop_prefix.3.14.pyc | Bin 0 -> 1285 bytes 8 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 tests/behavior/cases/b_conditional_loop_prefix.py create mode 100644 tests/fixtures/conditional_loop_prefix.py create mode 100644 tests/pyc/conditional_loop_prefix.3.11.pyc create mode 100644 tests/pyc/conditional_loop_prefix.3.12.pyc create mode 100644 tests/pyc/conditional_loop_prefix.3.13.pyc create mode 100644 tests/pyc/conditional_loop_prefix.3.14.pyc diff --git a/src/decompiler.rs b/src/decompiler.rs index 9e22e2f9..fa0ab533 100644 --- a/src/decompiler.rs +++ b/src/decompiler.rs @@ -23192,10 +23192,9 @@ impl<'a> Ctx<'a> { fn is_loop_top_prefix(&self, t: usize, start: usize) -> bool { // 3.11+ folds inline CACHE entries into each instruction's size, // so offsets inside a cached instruction have no idx_of entry - // either - a back edge landing <=8 bytes before the loop top - // would falsely match a CACHE tail (compileall 3.13 main tore - // into an INCOMPLETE stub). Modern offsets therefore need proof from - // original bytes, rather than absence from the instruction map. + // either. Absence from the instruction map cannot distinguish + // a CACHE tail from an EXTENDED_ARG prefix: modern offsets need + // proof from the original bytes. if t >= start || start - t > 8 { return false; } @@ -33234,6 +33233,10 @@ if split_cond { // `if c: continue`-style: fall through to the // arm below that opens an If ending at that back // edge instead of closing the loop here. + // A prefixed loop top has the same identity here + // as when matching the conditional above. Otherwise + // a later edge to EXTENDED_ARG is missed and a + // conditional continue prematurely closes the loop. let loop_start0 = self.blocks[i].start; let has_later_back_edge = self .idx_of @@ -33241,7 +33244,10 @@ if split_cond { .map_or(false, |&ci0| { self.instrs.iter().skip(ci0 + 1).any(|ins| { ins.is_backward - && ins.target == Some(loop_start0) + && ins.target.map_or(false, |t| { + t == loop_start0 + || self.is_loop_top_prefix(t, loop_start0) + }) && matches!( ins.op, Op::JUMP_ABSOLUTE @@ -33421,8 +33427,21 @@ if split_cond { } return; } + // A forward conditional can skip this back + // edge and land on the next one. Keep the + // skipped edge inside its fall-through arm; + // ending before it moves `continue` outside + // the guard and changes the recompiled jumps. + let body_end = if self.version.at_least(3, 11) + && raw_target == inst.end() + && self.is_loop_top_prefix(target, loop_start) + { + inst.end() + } else { + inst.start + }; let mut blk = - Block::new(BlockType::If, self.cur_next, inst.start); + Block::new(BlockType::If, self.cur_next, body_end); let c = if jump_if_true { negate_cond(cond.clone()) } else { diff --git a/tests/behavior/cases/b_conditional_loop_prefix.py b/tests/behavior/cases/b_conditional_loop_prefix.py new file mode 100644 index 00000000..2500953a --- /dev/null +++ b/tests/behavior/cases/b_conditional_loop_prefix.py @@ -0,0 +1,63 @@ +# MIN_VERSION: 3.11 + +# Attribute loads make FOR_ITER require EXTENDED_ARG on Python 3.12+. +# Conditional branches lead to adjacent back edges targeting that prefix. +def process(rows, work): + success = True + for row in rows: + if row.is_file: + if not work( + "file", row.ok, + row.a, row.b, row.c, row.d, row.e, row.f, + row.g, row.h, row.i, row.j, row.k, row.l, + ): + success = False + else: + if not work( + "directory", row.ok, + row.a, row.b, row.c, row.d, row.e, row.f, + row.g, row.h, row.i, row.j, row.k, row.l, + ): + success = False + return success + + +class Row: + a = 1 + b = 2 + c = 3 + d = 4 + e = 5 + f = 6 + g = 7 + h = 8 + i = 9 + j = 10 + k = 11 + l = 12 + + def __init__(self, is_file, ok): + self.is_file = is_file + self.ok = ok + + +def work(kind, ok, *values): + calls.append((kind, ok, values)) + return ok + + +for outcomes in [ + [], + [(True, True)], + [(True, False)], + [(False, True)], + [(False, False)], + [(True, True), (False, True)], + [(False, True), (True, True)], + [(True, False), (False, True), (True, True)], + [(False, False), (True, True), (False, True)], +]: + calls = [] + rows = [Row(is_file, ok) for is_file, ok in outcomes] + print(process(rows, work)) + print(calls) diff --git a/tests/fixtures/conditional_loop_prefix.py b/tests/fixtures/conditional_loop_prefix.py new file mode 100644 index 00000000..1a209679 --- /dev/null +++ b/tests/fixtures/conditional_loop_prefix.py @@ -0,0 +1,20 @@ +# Attribute loads make FOR_ITER require EXTENDED_ARG on Python 3.12+. +# Conditional branches lead to adjacent back edges targeting that prefix. +def process(rows, work): + success = True + for row in rows: + if row.is_file: + if not work( + "file", row.ok, + row.a, row.b, row.c, row.d, row.e, row.f, + row.g, row.h, row.i, row.j, row.k, row.l, + ): + success = False + else: + if not work( + "directory", row.ok, + row.a, row.b, row.c, row.d, row.e, row.f, + row.g, row.h, row.i, row.j, row.k, row.l, + ): + success = False + return success diff --git a/tests/integration.rs b/tests/integration.rs index 985d2196..8c65d128 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -183,3 +183,31 @@ fn extended_loop_prefix_preserves_handler_continue() { } } } + +#[test] +fn conditional_loop_prefix_keeps_both_arms_inside_loop() { + for version in ["3.11", "3.12", "3.13", "3.14"] { + let (src, clean) = decompile_fixture(&format!("conditional_loop_prefix.{version}.pyc")); + assert!(clean, "Python {version}: output marked incomplete:\n{src}"); + let args = "row.ok, row.a, row.b, row.c, row.d, row.e, row.f, row.g, row.h, row.i, row.j, row.k, row.l"; + let file_condition = if version == "3.12" { "" } else { "not " }; + let directory_condition = if version == "3.11" { "elif not" } else { "if" }; + assert_contains( + &src, + &[ + &format!(" for row in rows:\n if row.is_file:\n if {file_condition}work('file', {args}):"), + &format!(" {directory_condition} work('directory', {args}):"), + " return success", + ], + version, + ); + assert!(!src.contains("\n success = False"), "Python {version}: failure assignment escaped the loop:\n{src}"); + if version == "3.13" || version == "3.14" { + assert_contains( + &src, + &[" success = False\n continue\n continue"], + version, + ); + } + } +} diff --git a/tests/pyc/conditional_loop_prefix.3.11.pyc b/tests/pyc/conditional_loop_prefix.3.11.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a205329d5ebb6d2b9186367eda0dc3fe796ca138 GIT binary patch literal 949 zcmdUtJ8#oa6vxl;BW@CG+6GZjB;?f#9e}A73$HB_i!eXG+AmN@2uCiOmvMN^F&Qp~Te^*GgP3aXwUGbeDa?uN;7P^c|bj0)40* zu;SeY)%{=Z{yflVS7vKN8Qm~qH&rzV1@mJr_B}^gq3q;grSWZrn+hK)^c4mQnZmBZ zM+zS+3>9uEq=LvZlp*+@RK^|`+scxC-)B-PJrCx~a>4a=LhStWPTGV&l+E?Fq5?oh zxq127m-xmH(&NCP>PJX4!G;&`07_^omNNDL%#)T=^bd)W3oA+)HldCWQvemUtnQvM zk0g7mKpAa?M$gVQ944xboRgktc#@42(V3B+^hD*R&jr!hbBI!+Mj(v_(uF{}7)X}_ z>6Jiw?QeQA+!w>zk*!vg+R3158twRh)u>!lhksQ~sbg(Z){Xjm8kCg`n4$E2q2*l9 zxg##vE_1Ac&q}5tbeCr(7K*f@^=KbXG{4I5_<=u&b1^I5np;x1RcT(Q|II1f}wq#O2HGsfgGQ J&tq)h{R6uA+}i*E literal 0 HcmV?d00001 diff --git a/tests/pyc/conditional_loop_prefix.3.13.pyc b/tests/pyc/conditional_loop_prefix.3.13.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4ca9f1e926c273607984e3f8fc72e407bc33c14 GIT binary patch literal 1276 zcmeH{KX21O7{=eTowy{eN$aRBjX)7%zzZEvrXm(p!jcFP93e48!EuXeh@-nirCUa} zQV?n~rLaIJU_ku}e1!0-QU@lsL@8qBy^udu@sJO|4R_D|{P?}|qx1FW<{~Np{5Y%h?0v{6dO!PvwM#KDypqMIAb2k z>qAj|4`8#akWW25CR_Cr*b^J=nt9gEB?_=#|*7@70@5+ikfAxexX``{SKY>xb)~Zyu$-q^2_c*0s3N1M0dFNF-nDy{H5R z#Fsmc^lifO2=dmWW- z89?e*3PY1AAR(X=Fjc$(6LYv?;{_7JpOtf?IH^j8JOCc9zw`6wWBb$by=0tL@x>fS-Rsy&XP0(nS60C{o&(u%G0TmTcOWs~+7q|)ar7MPGF%Xa~M)lzRB zGl%kGS2(W$X22rnu`@c4_w726YT;e|Gty|k3Ki0kQ1{PBWBn>rNacr}IFK4A5SB>I zfpl^pO$?+{1L^cYdiGy>rTboCavKquTwd{&+z~+T6TYQPQRe+WMT`$~7$QXnXKMy$ zHFpO(S)#fJFbcC^DLCocEuj_N5?vOfj?1VWcx)$a2;JociG|{A3bHQ9eL-wN96_ic zML{+Mc_4@@$U{NUP)ch;W8O9w+P23wh0e>iO*t2;gmSv)=jY}`I-oq@3q|*FP-c`b z*xr`o2ClcYUMiG4&oB6l;>FAUj?jJP$qr18kNo-gZ(J(SFeWQEzMYQms*N#YFSD1~ z&3ug~8|GA1tw(Foh8aJI>_?vQgYtg4YS(FvzFd8~_G<0j_4nDs?1#mprH@NPnf~Zn z!uSEWG-HsOd#<;}P`3<7OgGF_b+vBQtf3kWW$HZbSMO;r^wv2r6U|u|k5^XO_~o#Y qCY>w}ft|sUC2e-mJl25# literal 0 HcmV?d00001