Teach InteractiveUtils how to read `ncat` AST #45399

BioTurboNick · 2022-05-20T17:33:01Z

Will resolve #43294

BioTurboNick · 2022-05-21T01:53:05Z

Well, that was a lot of work for an underwhelming result 😆

They all have the same entry point, so they all look like this:

julia> @code_lowered [3 4 ;;; 5 4]
CodeInfo(
1 ─ %1 = Core.tuple(dimsshape, row_first)
│   %2 = Core._apply_iterate(Base.iterate, Base._hvncat, %1, xs)
└──      return %2
)

The LLVM and assembly are more interesting

julia> @code_llvm [3 4 ;;; 5 4]
;  @ abstractarray.jl:2129 within `hvncat`
; Function Attrs: uwtable
define nonnull {}* @julia_hvncat_827([3 x i64]* nocapture nonnull readonly align 8 dereferenceable(24) %0, i8 zeroext %1, i64 signext %2, i64 signext %3, i64 signext %4, i64 signext %5) #0 {
top:
  %6 = alloca [7 x {}*], align 8
  %gcframe10 = alloca [7 x {}*], align 16
  %gcframe10.sub = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 0
  %.sub = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 0
  %7 = bitcast [7 x {}*]* %gcframe10 to i8*
  call void @llvm.memset.p0i8.i32(i8* nonnull align 16 dereferenceable(56) %7, i8 0, i32 56, i1 false)
  %8 = call {}*** inttoptr (i64 43440368 to {}*** ()*)() #4
; ┌ @ abstractarray.jl:2134 within `_hvncat`
   %9 = bitcast [7 x {}*]* %gcframe10 to i64*
   store i64 20, i64* %9, align 16
   %10 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 1
   %11 = bitcast {}** %10 to {}***
   %12 = load {}**, {}*** %8, align 8
   store {}** %12, {}*** %11, align 8
   %13 = bitcast {}*** %8 to {}***
   store {}** %gcframe10.sub, {}*** %13, align 8
   %ptls_field6 = getelementptr inbounds {}**, {}*** %8, i64 2305843009213693954
   %14 = bitcast {}*** %ptls_field6 to i8**
   %ptls_load78 = load i8*, i8** %14, align 8
   %15 = call noalias nonnull {}* @jl_gc_pool_alloc(i8* %ptls_load78, i32 1416, i32 32) #5
   %16 = bitcast {}* %15 to i64*
   %17 = getelementptr inbounds i64, i64* %16, i64 -1
   store atomic i64 289240400, i64* %17 unordered, align 8
   %18 = bitcast {}* %15 to i8*
   %19 = bitcast [3 x i64]* %0 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(24) %18, i8* nonnull align 8 dereferenceable(24) %19, i64 24, i1 false)
   %20 = and i8 %1, 1
   %.not = icmp eq i8 %20, 0
   %21 = select i1 %.not, {}* inttoptr (i64 286603920 to {}*), {}* inttoptr (i64 286531536 to {}*)
   %22 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 6
   store {}* %15, {}** %22, align 16
   %23 = call nonnull {}* @jl_box_int64(i64 signext %2)
   %24 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 5
   store {}* %23, {}** %24, align 8
   %25 = call nonnull {}* @jl_box_int64(i64 signext %3)
   %26 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 4
   store {}* %25, {}** %26, align 16
   %27 = call nonnull {}* @jl_box_int64(i64 signext %4)
   %28 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 3
   store {}* %27, {}** %28, align 8
   %29 = call nonnull {}* @jl_box_int64(i64 signext %5)
   %30 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 2
   store {}* %29, {}** %30, align 16
   store {}* inttoptr (i64 284363968 to {}*), {}** %.sub, align 8
   %31 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 1
   store {}* %15, {}** %31, align 8
   %32 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 2
   store {}* %21, {}** %32, align 8
   %33 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 3
   store {}* %23, {}** %33, align 8
   %34 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 4
   store {}* %25, {}** %34, align 8
   %35 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 5
   store {}* %27, {}** %35, align 8
   %36 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 6
   store {}* %29, {}** %36, align 8
   %37 = call nonnull {}* @jl_invoke({}* inttoptr (i64 352304192 to {}*), {}** nonnull %.sub, i32 7, {}* inttoptr (i64 206632560 to {}*))
   %38 = load {}*, {}** %10, align 8
   %39 = bitcast {}*** %8 to {}**
   store {}* %38, {}** %39, align 8
; └
  ret {}* %37
}

julia> @code_native [3 4 ;;; 5 4]
        .text
; ┌ @ abstractarray.jl:2129 within `hvncat`
        pushq   %rbp
        movq    %rsp, %rbp
        pushq   %r15
        pushq   %r14
        pushq   %r13
        pushq   %r12
        pushq   %rsi
        pushq   %rdi
        pushq   %rbx
        andq    $-32, %rsp
        subq    $192, %rsp
        movq    %r9, 56(%rsp)
        movq    %r8, %rsi
        movl    %edx, %ebx
        movq    %rcx, %r14
        vxorps  %xmm0, %xmm0, %xmm0
        vmovups %ymm0, 88(%rsp)
        vmovaps %ymm0, 64(%rsp)
        movl    $jl_get_pgcstack, %eax
        vzeroupper
        callq   *%rax
        movq    %rax, %r15
; │┌ @ abstractarray.jl:2134 within `_hvncat`
        movq    $20, 64(%rsp)
        movq    (%r15), %rax
        movq    %rax, 72(%rsp)
        leaq    64(%rsp), %rax
        movq    %rax, (%r15)
        movq    16(%r15), %rcx
        movabsq $jl_gc_pool_alloc, %rax
        movl    $1416, %edx                     # imm = 0x588
        movl    $32, %r8d
        callq   *%rax
        movq    %rax, %rdi
        movq    $289240400, -8(%rdi)            # imm = 0x113D7550
        movq    16(%r14), %rax
        movq    %rax, 16(%rdi)
        vmovups (%r14), %xmm0
        vmovups %xmm0, (%rdi)
        testb   $1, %bl
        movl    $286603920, %eax                # imm = 0x11153A90
        movl    $286531536, %r12d               # imm = 0x11141FD0
        cmoveq  %rax, %r12
        movq    %rdi, 112(%rsp)
        movabsq $jl_box_int64, %r13
        movq    %rsi, %rcx
        callq   *%r13
        movq    %rax, %r14
        movq    %r14, 104(%rsp)
        movq    56(%rsp), %rcx
        callq   *%r13
        movq    %rax, %rsi
        movq    %rsi, 96(%rsp)
        movq    48(%rbp), %rcx
        callq   *%r13
        movq    %rax, %rbx
        movq    %rbx, 88(%rsp)
        movq    56(%rbp), %rcx
        callq   *%r13
        movq    %rax, 80(%rsp)
        movq    $284363968, 128(%rsp)           # imm = 0x10F30CC0
        movq    %rdi, 136(%rsp)
        movq    %r12, 144(%rsp)
        movq    %r14, 152(%rsp)
        movq    %rsi, 160(%rsp)
        movq    %rbx, 168(%rsp)
        movq    %rax, 176(%rsp)
        movabsq $jl_invoke, %rax
        leaq    128(%rsp), %rdx
        movl    $352304192, %ecx                # imm = 0x14FFBC40
        movl    $206632560, %r9d                # imm = 0xC50F670
        movl    $7, %r8d
        callq   *%rax
        movq    72(%rsp), %rcx
        movq    %rcx, (%r15)
; │└
        leaq    -56(%rbp), %rsp
        popq    %rbx
        popq    %rdi
        popq    %rsi
        popq    %r12
        popq    %r13
        popq    %r14
        popq    %r15
        popq    %rbp
        retq
        nopl    (%rax,%rax)
; └

BioTurboNick · 2024-02-06T18:32:01Z

Might be good to get this out of the way and into 1.11, if there are no objections. @KristofferC

BioTurboNick · 2025-03-12T21:00:47Z

This should probably make it in at some point, right?

PallHaraldsson · 2025-10-15T13:55:40Z

Yes, merge it? Or are test failures real (and making this overlooked, rerun them?)? It seems like a false alarm on apple but not sure about the other, then REPL related (only on 32-bit, strange).

BioTurboNick · 2025-10-15T15:42:02Z

They both seem to be timeouts? The REPL error in the Linux 32-bit runner also occurs in 32-bit Windows CI but doesn't make it fail, so I think it's unrelated. I'll see if it's a known issue.

BioTurboNick · 2025-10-15T15:49:20Z

It's happening in other recent PRs: https://buildkite.com/julialang/julia-master/builds/51387/steps/canvas?sid=0199e85c-6990-43c3-bed9-f157220cbb02

adienes · 2025-10-15T16:09:26Z

I get this:

julia> @code_lowered [3 4 ;;; 5 4]
ERROR: LoadError: UndefVarError: `typesof` not defined in `InteractiveUtils`
Suggestion: check for spelling errors or missing imports.
Stacktrace:
 [1] gen_call_with_extracted_types(__module__::Module, fcn::Symbol, ex0::Expr, kws::Vector{…}; is_source_reflection::Bool, supports_binding_reflection::Bool, use_signature_tuple::Bool)
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:593
 [2] gen_call_with_extracted_types
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:386 [inlined]
 [3] #gen_call_with_extracted_types_and_kwargs#75
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:645 [inlined]
 [4] kwcall(::@NamedTuple{…}, ::typeof(InteractiveUtils.gen_call_with_extracted_types_and_kwargs), __module__::Module, fcn::Symbol, ex0::Tuple{…})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:631
 [5] var"@code_lowered"(__source__::LineNumberNode, __module__::Module, ex0::Vararg{Any})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:673
 [6] fl_lower
   @ ./flfrontend.jl:24 [inlined]
 [7] fl_lower(ex::Expr, mod::Module, filename::Ptr{UInt8}, lineno::UInt64)
   @ Base ./flfrontend.jl:23
in expression starting at REPL[2]:1
Some type information was truncated. Use `show(err)` to see complete types.

would it be possible to add a test case or two? maybe with ;; in different positions & combinations, mixing in spaces / commas, etc.

adienes · 2025-10-15T16:17:14Z

stdlib/InteractiveUtils/src/macros.jl

+                    if x.head === :nrow
+                        extract_elements.(x.args[2:end])
+                    elseif x.head === :row
+                        is_row_first = true


for is_row_first to work correctly here, it relies on extract_elements.(args) to evaluate left-to-right ? that seems fragile (and definitely confusing)

adienes · 2025-10-15T16:17:40Z

stdlib/InteractiveUtils/src/macros.jl

+                                map(esc, xs)...), kws...)
+            else
+                shape = get_shape(args, true, d)
+                is_balanced = sum(map((x, y) -> sum(map(z -> z - y, x)), shape[2:end], first.(shape[2:end]))) == 0


I don't know if I understand this yet. so given rows of length 2, 1, 3 that would be "balanced" ?

There are three method paths from the ncat AST:

Fully specified count of concat arguments at all levels, as a tuple of tuples of ints; the "shape" form

When the number of concat arguments within each dimension are the same, it is "balanced" and simplified to a tuple of ints counting the concat arguments along each dimension; the "dims" form

When there's only one dimension, it's simplified to an int indicating which dimension to concat.

Balanced: [a b c;;; d e f]
Not balanced: [a b c;;; d e]

BioTurboNick · 2025-10-15T16:57:23Z

I get this:

julia> @code_lowered [3 4 ;;; 5 4]
ERROR: LoadError: UndefVarError: `typesof` not defined in `InteractiveUtils`
Suggestion: check for spelling errors or missing imports.
Stacktrace:
 [1] gen_call_with_extracted_types(__module__::Module, fcn::Symbol, ex0::Expr, kws::Vector{…}; is_source_reflection::Bool, supports_binding_reflection::Bool, use_signature_tuple::Bool)
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:593
 [2] gen_call_with_extracted_types
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:386 [inlined]
 [3] #gen_call_with_extracted_types_and_kwargs#75
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:645 [inlined]
 [4] kwcall(::@NamedTuple{…}, ::typeof(InteractiveUtils.gen_call_with_extracted_types_and_kwargs), __module__::Module, fcn::Symbol, ex0::Tuple{…})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:631
 [5] var"@code_lowered"(__source__::LineNumberNode, __module__::Module, ex0::Vararg{Any})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:673
 [6] fl_lower
   @ ./flfrontend.jl:24 [inlined]
 [7] fl_lower(ex::Expr, mod::Module, filename::Ptr{UInt8}, lineno::UInt64)
   @ Base ./flfrontend.jl:23
in expression starting at REPL[2]:1
Some type information was truncated. Use `show(err)` to see complete types.

would it be possible to add a test case or two? maybe with ;; in different positions & combinations, mixing in spaces / commas, etc.

Seems that this commit in master, but not 1.12, broke it: c3e7b1b

I'll have to adjust.

BioTurboNick added 4 commits May 20, 2022 13:27

Initial work

4b5f768

Shape variant

d5e03d2

Shape variant

2c79364

Rest of the way.

b0814fb

Merge branch 'master' into hvncat-interactiveutils

322a005

BioTurboNick marked this pull request as ready for review May 22, 2022 01:29

BioTurboNick changed the title ~~Draft: Teach InteractiveUtils how to read ncat AST~~ Teach InteractiveUtils how to read ncat AST May 22, 2022

BioTurboNick added 8 commits May 27, 2022 11:45

Merge branch 'master' into hvncat-interactiveutils

25fceb4

Merge branch 'master' into hvncat-interactiveutils

e60228d

Merge branch 'master' into hvncat-interactiveutils

ad9355b

Remove what I think is a stray debug block?

878736b

Merge branch 'master' into hvncat-interactiveutils

aa5507f

Merge branch 'master' into hvncat-interactiveutils

b5eab55

Merge branch 'master' into hvncat-interactiveutils

af7275f

Merge branch 'master' into hvncat-interactiveutils

c31ad55

BioTurboNick added 2 commits January 15, 2025 14:01

Merge branch 'master' into hvncat-interactiveutils

fce0ee6

Merge branch 'master' into hvncat-interactiveutils

544fa0c

BioTurboNick added 2 commits June 3, 2025 16:28

Merge branch 'master' into hvncat-interactiveutils

9db1f0a

Merge branch 'master' into hvncat-interactiveutils

1c786c4

adienes reviewed Oct 15, 2025

View reviewed changes

Attempt to switch to typesof_expr

666251f

fix

21ced5c

Uh oh!

Teach InteractiveUtils how to read ncat AST #45399

Are you sure you want to change the base?

Teach InteractiveUtils how to read ncat AST #45399

Conversation

BioTurboNick commented May 20, 2022

Uh oh!

BioTurboNick commented May 21, 2022 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

BioTurboNick commented Feb 6, 2024

Uh oh!

BioTurboNick commented Mar 12, 2025

Uh oh!

PallHaraldsson commented Oct 15, 2025

Uh oh!

BioTurboNick commented Oct 15, 2025

Uh oh!

BioTurboNick commented Oct 15, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

adienes commented Oct 15, 2025

Uh oh!

adienes Oct 15, 2025

Choose a reason for hiding this comment

Uh oh!

adienes Oct 15, 2025

Choose a reason for hiding this comment

Uh oh!

BioTurboNick Oct 15, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

BioTurboNick Oct 15, 2025

Choose a reason for hiding this comment

Uh oh!

BioTurboNick commented Oct 15, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

Teach InteractiveUtils how to read `ncat` AST #45399

Teach InteractiveUtils how to read `ncat` AST #45399

BioTurboNick commented May 21, 2022 •

edited

Loading

BioTurboNick commented Oct 15, 2025 •

edited

Loading

BioTurboNick Oct 15, 2025 •

edited

Loading