8335256: [lworld] C2: Remove larval InlineTypeNode

8325627: [lworld] C2 compilation bailout for TestNullableInlineTypes::test85
8354283: [lworld] TestAllocationMergeAndFolding fails since jdk-25+16, since an allocation is detected

Reviewed-by: thartmann

Author: Quan Anh Mai

src/hotspot/share/opto/callGenerator.cpp

@@ -722,7 +722,7 @@ void CallGenerator::do_late_inline_helper() {
     Node* buffer_oop = nullptr;
     ciMethod* inline_method = inline_cg()->method();
     ciType* return_type = inline_method->return_type();
-    if (!call->tf()->returns_inline_type_as_fields() && is_mh_late_inline() &&
+    if (!call->tf()->returns_inline_type_as_fields() &&
         return_type->is_inlinetype() && return_type->as_inline_klass()->can_be_returned_as_fields()) {
       // Allocate a buffer for the inline type returned as fields because the caller expects an oop return.
       // Do this before the method handle call in case the buffer allocation triggers deoptimization and

src/hotspot/share/opto/compile.cpp

@@ -2843,16 +2843,39 @@ void Compile::Optimize() {
 
   if (failing())  return;
 
+  {
+    // Eliminate some macro nodes before EA to reduce analysis pressure
+    PhaseMacroExpand mexp(igvn);
+    mexp.eliminate_macro_nodes();
+    if (failing()) {
+      return;
+    }
+    igvn.set_delay_transform(false);
+    print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
+  }
+
   // Perform escape analysis
   if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
     if (has_loops()) {
       // Cleanup graph (remove dead nodes).
       TracePhase tp(_t_idealLoop);
       PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
-      if (failing())  return;
+      if (failing()) {
+        return;
+      }
+      print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
+
+      // Eliminate some macro nodes before EA to reduce analysis pressure
+      PhaseMacroExpand mexp(igvn);
+      mexp.eliminate_macro_nodes();
+      if (failing()) {
+        return;
+      }
+      igvn.set_delay_transform(false);
+      print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
     }
+
     bool progress;
-    print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
     do {
       ConnectionGraph::do_analysis(this, &igvn);
 
@@ -2870,12 +2893,10 @@ void Compile::Optimize() {
         TracePhase tp(_t_macroEliminate);
         PhaseMacroExpand mexp(igvn);
         mexp.eliminate_macro_nodes();
-        if (failing()) return;
-
+        if (failing()) {
+          return;
+        }
         igvn.set_delay_transform(false);
-        igvn.optimize();
-        if (failing()) return;
-
         print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
       }
 
@@ -2976,8 +2997,14 @@ void Compile::Optimize() {
 
   {
     TracePhase tp(_t_macroExpand);
+    PhaseMacroExpand mex(igvn);
+    // Last attempt to eliminate macro nodes.
+    mex.eliminate_macro_nodes();
+    if (failing()) {
+      return;
+    }
+
     print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
-    PhaseMacroExpand  mex(igvn);
     if (mex.expand_macro_nodes()) {
       assert(failing(), "must bail out w/ explicit message");
       return;

src/hotspot/share/opto/doCall.cpp

@@ -25,6 +25,7 @@
 #include "ci/ciCallSite.hpp"
 #include "ci/ciMethodHandle.hpp"
 #include "ci/ciSymbols.hpp"
+#include "classfile/vmIntrinsics.hpp"
 #include "classfile/vmSymbols.hpp"
 #include "compiler/compileBroker.hpp"
 #include "compiler/compileLog.hpp"
@@ -88,6 +89,64 @@ static void trace_type_profile(Compile* C, ciMethod* method, JVMState* jvms,
   }
 }
 
+static bool arg_can_be_larval(ciMethod* callee, int arg_idx) {
+  if (callee->is_object_constructor() && arg_idx == 0) {
+    return true;
+  }
+
+  if (arg_idx != 1 || callee->intrinsic_id() == vmIntrinsicID::_none) {
+    return false;
+  }
+
+  switch (callee->intrinsic_id()) {
+    case vmIntrinsicID::_finishPrivateBuffer:
+    case vmIntrinsicID::_putBoolean:
+    case vmIntrinsicID::_putBooleanOpaque:
+    case vmIntrinsicID::_putBooleanRelease:
+    case vmIntrinsicID::_putBooleanVolatile:
+    case vmIntrinsicID::_putByte:
+    case vmIntrinsicID::_putByteOpaque:
+    case vmIntrinsicID::_putByteRelease:
+    case vmIntrinsicID::_putByteVolatile:
+    case vmIntrinsicID::_putChar:
+    case vmIntrinsicID::_putCharOpaque:
+    case vmIntrinsicID::_putCharRelease:
+    case vmIntrinsicID::_putCharUnaligned:
+    case vmIntrinsicID::_putCharVolatile:
+    case vmIntrinsicID::_putShort:
+    case vmIntrinsicID::_putShortOpaque:
+    case vmIntrinsicID::_putShortRelease:
+    case vmIntrinsicID::_putShortUnaligned:
+    case vmIntrinsicID::_putShortVolatile:
+    case vmIntrinsicID::_putInt:
+    case vmIntrinsicID::_putIntOpaque:
+    case vmIntrinsicID::_putIntRelease:
+    case vmIntrinsicID::_putIntUnaligned:
+    case vmIntrinsicID::_putIntVolatile:
+    case vmIntrinsicID::_putLong:
+    case vmIntrinsicID::_putLongOpaque:
+    case vmIntrinsicID::_putLongRelease:
+    case vmIntrinsicID::_putLongUnaligned:
+    case vmIntrinsicID::_putLongVolatile:
+    case vmIntrinsicID::_putFloat:
+    case vmIntrinsicID::_putFloatOpaque:
+    case vmIntrinsicID::_putFloatRelease:
+    case vmIntrinsicID::_putFloatVolatile:
+    case vmIntrinsicID::_putDouble:
+    case vmIntrinsicID::_putDoubleOpaque:
+    case vmIntrinsicID::_putDoubleRelease:
+    case vmIntrinsicID::_putDoubleVolatile:
+    case vmIntrinsicID::_putReference:
+    case vmIntrinsicID::_putReferenceOpaque:
+    case vmIntrinsicID::_putReferenceRelease:
+    case vmIntrinsicID::_putReferenceVolatile:
+    case vmIntrinsicID::_putValue:
+      return true;
+    default:
+      return false;
+  }
+}
+
 CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool call_does_dispatch,
                                        JVMState* jvms, bool allow_inline,
                                        float prof_factor, ciKlass* speculative_receiver_type,
@@ -645,6 +704,15 @@ void Parse::do_call() {
     set_stack(sp() - nargs, casted_receiver);
   }
 
+  // Scalarize value objects passed into this invocation if we know that they are not larval
+  for (int arg_idx = 0; arg_idx < nargs; arg_idx++) {
+    if (arg_can_be_larval(callee, arg_idx)) {
+      continue;
+    }
+
+    cast_to_non_larval(peek(nargs - 1 - arg_idx));
+  }
+
   // Note:  It's OK to try to inline a virtual call.
   // The call generator will not attempt to inline a polymorphic call
   // unless it knows how to optimize the receiver dispatch.
@@ -807,59 +875,27 @@ void Parse::do_call() {
     if (is_reference_type(ct)) {
       record_profiled_return_for_speculation();
     }
-    if (rtype->is_inlinetype() && !peek()->is_InlineType()) {
-      Node* retnode = pop();
-      retnode = InlineTypeNode::make_from_oop(this, retnode, rtype->as_inline_klass());
-      push_node(T_OBJECT, retnode);
-    }
 
-    // Note that:
-    // - The caller map is the state just before the call of the currently parsed method with all arguments
-    //   on the stack. Therefore, we have caller_map->arg(0) == this.
-    // - local(0) contains the updated receiver after calling an inline type constructor.
-    // - Abstract value classes are not ciInlineKlass instances and thus abstract_value_klass->is_inlinetype() is false.
-    //   We use the bottom type of the receiver node to determine if we have a value class or not.
-    const bool is_current_method_inline_type_constructor =
-        // Is current method a constructor (i.e <init>)?
-        _method->is_object_constructor() &&
-        // Is the holder of the current constructor method an inline type?
-        _caller->map()->argument(_caller, 0)->bottom_type()->is_inlinetypeptr();
-    assert(!is_current_method_inline_type_constructor || !cg->method()->is_object_constructor() || receiver != nullptr,
-           "must have valid receiver after calling another constructor");
-    if (is_current_method_inline_type_constructor &&
-        // Is the just called method an inline type constructor?
-        cg->method()->is_object_constructor() && receiver->bottom_type()->is_inlinetypeptr() &&
-         // AND:
-         // 1) ... invoked on the same receiver? Then it's another constructor on the same object doing the initialization.
-        (receiver == _caller->map()->argument(_caller, 0) ||
-         // 2) ... abstract? Then it's the call to the super constructor which eventually calls Object.<init> to
-         //                    finish the initialization of this larval.
-         cg->method()->holder()->is_abstract() ||
-         // 3) ... Object.<init>? Then we know it's the final call to finish the larval initialization. Other
-         //        Object.<init> calls would have a non-inline-type receiver which we already excluded in the check above.
-         cg->method()->holder()->is_java_lang_Object())
-        ) {
-      assert(local(0)->is_InlineType() && receiver->bottom_type()->is_inlinetypeptr() && receiver->is_InlineType() &&
-             _caller->map()->argument(_caller, 0)->bottom_type()->inline_klass() == receiver->bottom_type()->inline_klass(),
-             "Unexpected receiver");
-      InlineTypeNode* updated_receiver = local(0)->as_InlineType();
-      InlineTypeNode* cloned_updated_receiver = updated_receiver->clone_if_required(&_gvn, _map);
-      cloned_updated_receiver->set_is_larval(false);
-      cloned_updated_receiver = _gvn.transform(cloned_updated_receiver)->as_InlineType();
-      // Receiver updated by the just called constructor. We need to update the map to make the effect visible. After
-      // the super() call, only the updated receiver in local(0) will be used from now on. Therefore, we do not need
-      // to update the original receiver 'receiver' but only the 'updated_receiver'.
-      replace_in_map(updated_receiver, cloned_updated_receiver);
-
-      if (_caller->has_method()) {
-        // If the current method is inlined, we also need to update the exit map to propagate the updated receiver
-        // to the caller map.
-        Node* receiver_in_caller = _caller->map()->argument(_caller, 0);
-        assert(receiver_in_caller->bottom_type()->inline_klass() == receiver->bottom_type()->inline_klass(),
-               "Receiver type mismatch");
-        _exits.map()->replace_edge(receiver_in_caller, cloned_updated_receiver, &_gvn);
+    if (!rtype->is_void() && cg->method()->intrinsic_id() != vmIntrinsicID::_makePrivateBuffer) {
+      Node* retnode = peek();
+      const Type* rettype = gvn().type(retnode);
+      if (rettype->is_inlinetypeptr() && !retnode->is_InlineType()) {
+        retnode = InlineTypeNode::make_from_oop(this, retnode, rettype->inline_klass());
+        dec_sp(1);
+        push(retnode);
       }
     }
+
+    if (cg->method()->is_object_constructor() && receiver != nullptr && gvn().type(receiver)->is_inlinetypeptr()) {
+      InlineTypeNode* non_larval = InlineTypeNode::make_from_oop(this, receiver, gvn().type(receiver)->inline_klass());
+      // Relinquish the oop input, we will delay the allocation to the point it is needed, see the
+      // comments in InlineTypeNode::Ideal for more details
+      non_larval = non_larval->clone_if_required(&gvn(), nullptr);
+      non_larval->set_oop(gvn(), null());
+      non_larval->set_is_buffered(gvn(), false);
+      non_larval = gvn().transform(non_larval)->as_InlineType();
+      map()->replace_edge(receiver, non_larval);
+    }
   }
 
   // Restart record of parsing work after possible inlining of call

src/hotspot/share/opto/graphKit.cpp

@@ -990,14 +990,7 @@ void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
     out_jvms->set_locoff(p);
     if (!can_prune_locals) {
       for (j = 0; j < l; j++) {
-        Node* val = in_map->in(k + j);
-        // Check if there's a larval that has been written in the callee state (constructor) and update it in the caller state
-        if (callee_jvms != nullptr && val->is_InlineType() && val->as_InlineType()->is_larval() &&
-            callee_jvms->method()->is_object_constructor() && val == in_map->argument(in_jvms, 0) &&
-            val->bottom_type()->is_inlinetypeptr()) {
-          val = callee_jvms->map()->local(callee_jvms, 0); // Receiver
-        }
-        call->set_req(p++, val);
+        call->set_req(p++, in_map->in(k + j));
       }
     } else {
       p += l;  // already set to top above by add_req_batch
@@ -1009,14 +1002,7 @@ void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
     out_jvms->set_stkoff(p);
     if (!can_prune_locals) {
       for (j = 0; j < l; j++) {
-        Node* val = in_map->in(k + j);
-        // Check if there's a larval that has been written in the callee state (constructor) and update it in the caller state
-        if (callee_jvms != nullptr && val->is_InlineType() && val->as_InlineType()->is_larval() &&
-            callee_jvms->method()->is_object_constructor() && val == in_map->argument(in_jvms, 0) &&
-            val->bottom_type()->is_inlinetypeptr()) {
-          val = callee_jvms->map()->local(callee_jvms, 0); // Receiver
-        }
-        call->set_req(p++, val);
+        call->set_req(p++, in_map->in(k + j));
       }
     } else if (can_prune_locals && stack_slots_not_pruned != 0) {
       // Divide stack into {S0,...,S1}, where S0 is set to top.
@@ -1515,6 +1501,17 @@ Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
   return cast;                  // Return casted value
 }
 
+Node* GraphKit::cast_to_non_larval(Node* obj) {
+  const Type* obj_type = gvn().type(obj);
+  if (obj->is_InlineType() || !obj_type->is_inlinetypeptr()) {
+    return obj;
+  }
+
+  Node* new_obj = InlineTypeNode::make_from_oop(this, obj, obj_type->inline_klass());
+  replace_in_map(obj, new_obj);
+  return new_obj;
+}
+
 // Sometimes in intrinsics, we implicitly know an object is not null
 // (there's no actual null check) so we can cast it to not null. In
 // the course of optimizations, the input to the cast can become null.
@@ -1920,28 +1917,7 @@ void GraphKit::set_arguments_for_java_call(CallJavaNode* call, bool is_late_inli
       continue;
     } else if (arg->is_InlineType()) {
       // Pass inline type argument via oop to callee
-      InlineTypeNode* inline_type = arg->as_InlineType();
-      const ciMethod* method = call->method();
-      ciInstanceKlass* holder = method->holder();
-      const bool is_receiver = (i == TypeFunc::Parms);
-      const bool is_abstract_or_object_klass_constructor = method->is_object_constructor() &&
-                                                           (holder->is_abstract() || holder->is_java_lang_Object());
-      const bool is_larval_receiver_on_super_constructor = is_receiver && is_abstract_or_object_klass_constructor;
-      bool must_init_buffer = true;
-      // We always need to buffer inline types when they are escaping. However, we can skip the actual initialization
-      // of the buffer if the inline type is a larval because we are going to update the buffer anyway which requires
-      // us to create a new one. But there is one special case where we are still required to initialize the buffer:
-      // When we have a larval receiver invoked on an abstract (value class) constructor or the Object constructor (that
-      // is not going to be inlined). After this call, the larval is completely initialized and thus not a larval anymore.
-      // We therefore need to force an initialization of the buffer to not lose all the field writes so far in case the
-      // buffer needs to be used (e.g. to read from when deoptimizing at runtime) or further updated in abstract super
-      // value class constructors which could have more fields to be initialized. Note that we do not need to
-      // initialize the buffer when invoking another constructor in the same class on a larval receiver because we
-      // have not initialized any fields, yet (this is done completely by the other constructor call).
-      if (inline_type->is_larval() && !is_larval_receiver_on_super_constructor) {
-        must_init_buffer = false;
-      }
-      arg = inline_type->buffer(this, true, must_init_buffer);
+      arg = arg->as_InlineType()->buffer(this, true);
     }
     if (t != Type::HALF) {
       arg_num++;
@@ -2018,20 +1994,6 @@ Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_p
     }
   }
 
-  // We just called the constructor on a value type receiver. Reload it from the buffer
-  ciMethod* method = call->method();
-  if (method->is_object_constructor() && !method->holder()->is_java_lang_Object()) {
-    InlineTypeNode* inline_type_receiver = call->in(TypeFunc::Parms)->isa_InlineType();
-    if (inline_type_receiver != nullptr) {
-      assert(inline_type_receiver->is_larval(), "must be larval");
-      assert(inline_type_receiver->is_allocated(&gvn()), "larval must be buffered");
-      InlineTypeNode* reloaded = InlineTypeNode::make_from_oop(this, inline_type_receiver->get_oop(),
-                                                               inline_type_receiver->bottom_type()->inline_klass());
-      assert(!reloaded->is_larval(), "should not be larval anymore");
-      replace_in_map(inline_type_receiver, reloaded);
-    }
-  }
-
   return ret;
 }
 
@@ -3458,7 +3420,7 @@ Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replac
 // If failure_control is supplied and not null, it is filled in with
 // the control edge for the cast failure.  Otherwise, an appropriate
 // uncommon trap or exception is thrown.
-Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node* *failure_control, bool null_free) {
+Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node* *failure_control, bool null_free, bool maybe_larval) {
   kill_dead_locals();           // Benefit all the uncommon traps
   const TypeKlassPtr* klass_ptr_type = _gvn.type(superklass)->is_klassptr();
   const Type* obj_type = _gvn.type(obj);
@@ -3474,6 +3436,9 @@ Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node* *failure_contro
     return obj;
   }
 
+  // Else it must be a non-larval object
+  obj = cast_to_non_larval(obj);
+
   const TypeKlassPtr* improved_klass_ptr_type = klass_ptr_type->try_improve();
   const TypeOopPtr* toop = improved_klass_ptr_type->cast_to_exactness(false)->as_instance_type();
   bool safe_for_replace = (failure_control == nullptr);
@@ -3704,7 +3669,7 @@ Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node* *failure_contro
 
   if (!stopped() && !res->is_InlineType()) {
     res = record_profiled_receiver_for_speculation(res);
-    if (toop->is_inlinetypeptr()) {
+    if (toop->is_inlinetypeptr() && !maybe_larval) {
       Node* vt = InlineTypeNode::make_from_oop(this, res, toop->inline_klass());
       res = vt;
       if (safe_for_replace) {