8355397: [lworld] Parse::array_load fails with "array can't be flat" assert

Reviewed-by: thartmann

Author: Quan Anh Mai

src/hotspot/share/c1/c1_Instruction.cpp

@@ -140,7 +140,7 @@ bool Instruction::maybe_flat_array() {
       if (type->is_obj_array_klass()) {
         // Due to array covariance, the runtime type might be a flat array.
         ciKlass* element_klass = type->as_obj_array_klass()->element_klass();
-        if (element_klass->can_be_inline_klass() && (!element_klass->is_inlinetype() || element_klass->as_inline_klass()->flat_in_array())) {
+        if (element_klass->can_be_inline_klass() && (!element_klass->is_inlinetype() || element_klass->as_inline_klass()->maybe_flat_in_array())) {
           return true;
         }
       } else if (type->is_flat_array_klass()) {

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -1827,7 +1827,7 @@ bool LIRGenerator::needs_flat_array_store_check(StoreIndexed* x) {
     ciType* type = x->value()->declared_type();
     if (type != nullptr && type->is_klass()) {
       ciKlass* klass = type->as_klass();
-      if (!klass->can_be_inline_klass() || (klass->is_inlinetype() && !klass->as_inline_klass()->flat_in_array())) {
+      if (!klass->can_be_inline_klass() || (klass->is_inlinetype() && !klass->as_inline_klass()->maybe_flat_in_array())) {
         // This is known to be a non-flat object. If the array is a flat array,
         // it will be caught by the code generated by array_store_check().
         return false;

src/hotspot/share/ci/ciArrayKlass.cpp

@@ -114,7 +114,7 @@ ciArrayKlass* ciArrayKlass::make(ciType* element_type, bool flat, bool null_free
       EXCEPTION_CONTEXT;
       Klass* ak = nullptr;
       InlineKlass* vk = InlineKlass::cast(klass->get_Klass());
-      if (flat && vk->flat_array()) {
+      if (flat && vk->maybe_flat_in_array()) {
         LayoutKind lk;
         if (null_free) {
           if (vk->is_naturally_atomic()) {

src/hotspot/share/ci/ciInlineKlass.cpp

@@ -33,8 +33,8 @@ int ciInlineKlass::payload_offset() const {
 }
 
 // Are arrays containing this inline type flat arrays?
-bool ciInlineKlass::flat_in_array() const {
-  GUARDED_VM_ENTRY(return to_InlineKlass()->flat_array();)
+bool ciInlineKlass::maybe_flat_in_array() const {
+  GUARDED_VM_ENTRY(return to_InlineKlass()->maybe_flat_in_array();)
 }
 
 // Can this inline type be passed as multiple values?

src/hotspot/share/ci/ciInlineKlass.hpp

@@ -60,7 +60,7 @@ class ciInlineKlass : public ciInstanceKlass {
   // Inline type fields
   int payload_offset() const;
 
-  bool flat_in_array() const;
+  bool maybe_flat_in_array() const;
   bool can_be_passed_as_fields() const;
   bool can_be_returned_as_fields() const;
   bool is_empty();

src/hotspot/share/ci/ciMetadata.hpp

@@ -61,7 +61,7 @@ class ciMetadata: public ciBaseObject {
   virtual bool is_obj_array_klass() const   { return false; }
   virtual bool is_type_array_klass() const  { return false; }
   virtual bool is_wrapper() const           { return false; }
-  virtual bool flat_in_array() const        { return false; }
+  virtual bool maybe_flat_in_array() const  { return false; }
   virtual void dump_replay_data(outputStream* st) { /* do nothing */ }
 
   ciMethod*                as_method() {

src/hotspot/share/oops/inlineKlass.cpp

@@ -284,22 +284,14 @@ void InlineKlass::write_value_to_addr(oop src, void* dst, LayoutKind lk, bool de
 
 // Arrays of...
 
-bool InlineKlass::flat_array() {
+bool InlineKlass::maybe_flat_in_array() {
   if (!UseArrayFlattening) {
     return false;
   }
   // Too many embedded oops
   if ((FlatArrayElementMaxOops >= 0) && (nonstatic_oop_count() > FlatArrayElementMaxOops)) {
     return false;
   }
-  // Declared atomic but not naturally atomic.
-  if (must_be_atomic() && !is_naturally_atomic()) {
-    return false;
-  }
-  // VM enforcing AlwaysAtomicAccess only...
-  if (AlwaysAtomicAccesses && (!is_naturally_atomic())) {
-    return false;
-  }
   // No flat layout?
   if (!has_nullable_atomic_layout() && !has_atomic_layout() && !has_non_atomic_layout()) {
     return false;

src/hotspot/share/oops/inlineKlass.hpp

@@ -254,7 +254,7 @@ class InlineKlass: public InstanceKlass {
 
   address payload_addr(oop o) const;
 
-  bool flat_array();
+  bool maybe_flat_in_array();
 
   bool contains_oops() const { return nonstatic_oop_map_count() > 0; }
   int nonstatic_oop_count();

src/hotspot/share/opto/graphKit.cpp

@@ -3663,7 +3663,7 @@ Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node* *failure_contro
   record_for_igvn(region);
 
   bool not_inline = !toop->can_be_inline_type();
-  bool not_flat_in_array = !UseArrayFlattening || not_inline || (toop->is_inlinetypeptr() && !toop->inline_klass()->flat_in_array());
+  bool not_flat_in_array = !UseArrayFlattening || not_inline || (toop->is_inlinetypeptr() && !toop->inline_klass()->maybe_flat_in_array());
   if (EnableValhalla && (not_inline || not_flat_in_array)) {
     // Check if obj has been loaded from an array
     obj = obj->isa_DecodeN() ? obj->in(1) : obj;
@@ -4008,7 +4008,7 @@ Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
     if (UseArrayFlattening && !xklass && ary_type != nullptr && !ary_type->is_null_free()) {
       // Don't constant fold if the runtime type might be a flat array but the static type is not.
       const TypeOopPtr* elem = ary_type->elem()->make_oopptr();
-      can_be_flat = ary_type->can_be_inline_array() && (!elem->is_inlinetypeptr() || elem->inline_klass()->flat_in_array());
+      can_be_flat = ary_type->can_be_inline_array() && (!elem->is_inlinetypeptr() || elem->inline_klass()->maybe_flat_in_array());
     }
     if (!can_be_flat && (xklass || (klass_t->isa_aryklassptr() && klass_t->is_aryklassptr()->elem() != Type::BOTTOM))) {
       jint lhelper;

src/hotspot/share/opto/library_call.cpp

@@ -4582,7 +4582,7 @@ bool LibraryCallKit::inline_newArray(bool null_free, bool atomic) {
       ciType* t = tp->java_mirror_type();
       if (t != nullptr && t->is_inlinetype()) {
         ciInlineKlass* vk = t->as_inline_klass();
-        bool flat = vk->flat_in_array();
+        bool flat = vk->maybe_flat_in_array();
         if (flat && atomic) {
           // Only flat if we have a corresponding atomic layout
           flat = null_free ? vk->has_atomic_layout() : vk->has_nullable_atomic_layout();

src/hotspot/share/opto/macro.cpp

@@ -954,7 +954,7 @@ SafePointScalarObjectNode* PhaseMacroExpand::create_scalarized_object_descriptio
     const TypeOopPtr* field_addr_type = res_type->add_offset(offset)->isa_oopptr();
     if (res_type->is_flat()) {
       ciInlineKlass* inline_klass = res_type->is_aryptr()->elem()->inline_klass();
-      assert(inline_klass->flat_in_array(), "must be flat in array");
+      assert(inline_klass->maybe_flat_in_array(), "must be flat in array");
       field_val = inline_type_from_mem(sfpt->memory(), sfpt->control(), inline_klass, field_addr_type->isa_aryptr(), 0, alloc);
     } else {
       field_val = value_from_mem(sfpt->memory(), sfpt->control(), basic_elem_type, field_type, field_addr_type, alloc);

src/hotspot/share/opto/parse2.cpp

@@ -85,7 +85,7 @@ void Parse::array_load(BasicType bt) {
   if (!array_type->is_not_flat()) {
     // Cannot statically determine if array is a flat array, emit runtime check
     assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() &&
-           (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->flat_in_array()), "array can't be flat");
+           (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat");
     IdealKit ideal(this);
     IdealVariable res(ideal);
     ideal.declarations_done();
@@ -228,7 +228,7 @@ void Parse::array_store(BasicType bt) {
     bool not_inline = !stored_value_casted_type->maybe_null() && !stored_value_casted_type->is_oopptr()->can_be_inline_type();
     bool not_null_free = not_inline;
     bool not_flat = not_inline || ( stored_value_casted_type->is_inlinetypeptr() &&
-                                   !stored_value_casted_type->inline_klass()->flat_in_array());
+                                   !stored_value_casted_type->inline_klass()->maybe_flat_in_array());
     if (!array_type->is_not_null_free() && not_null_free) {
       // Storing a non-inline type, mark array as not null-free.
       array_type = array_type->cast_to_not_null_free();

src/hotspot/share/opto/type.cpp

@@ -4077,7 +4077,7 @@ const TypeOopPtr* TypeOopPtr::make_from_klass_common(ciKlass *klass, bool klass_
     }
     bool not_inline = !exact_etype->can_be_inline_type();
     bool not_null_free = not_inline;
-    bool not_flat = !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->flat_in_array());
+    bool not_flat = !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->maybe_flat_in_array());
     // Even though MyValue is final, [LMyValue is not exact because null-free [LMyValue is a subtype.
     bool xk = etype->klass_is_exact() && !etype->is_inlinetypeptr();
     const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS, /* stable= */ false, /* flat= */ false, not_flat, not_null_free);
@@ -4387,7 +4387,7 @@ TypeInstPtr::TypeInstPtr(PTR ptr, ciKlass* k, const TypeInterfaces* interfaces,
   assert(k != nullptr &&
          (k->is_loaded() || o == nullptr),
          "cannot have constants with non-loaded klass");
-  assert(!klass()->flat_in_array() || flat_in_array, "Should be flat in array");
+  assert(!klass()->maybe_flat_in_array() || flat_in_array, "Should be flat in array");
   assert(!flat_in_array || can_be_inline_type(), "Only inline types can be flat in array");
 };
 
@@ -4421,7 +4421,7 @@ const TypeInstPtr *TypeInstPtr::make(PTR ptr,
   }
 
   // Check if this type is known to be flat in arrays
-  flat_in_array = flat_in_array || k->flat_in_array();
+  flat_in_array = flat_in_array || k->maybe_flat_in_array();
 
   // Now hash this baby
   TypeInstPtr *result =
@@ -6447,7 +6447,7 @@ uint TypeInstKlassPtr::hash(void) const {
 }
 
 const TypeInstKlassPtr *TypeInstKlassPtr::make(PTR ptr, ciKlass* k, const TypeInterfaces* interfaces, Offset offset, bool flat_in_array) {
-  flat_in_array = flat_in_array || k->flat_in_array();
+  flat_in_array = flat_in_array || k->maybe_flat_in_array();
 
   TypeInstKlassPtr *r =
     (TypeInstKlassPtr*)(new TypeInstKlassPtr(ptr, k, interfaces, offset, flat_in_array))->hashcons();
@@ -6820,7 +6820,7 @@ const TypeAryKlassPtr* TypeAryKlassPtr::make(PTR ptr, ciKlass* k, Offset offset,
   bool not_flat = (ptr == Constant) ? !flat : (!UseArrayFlattening || not_inline ||
                    (k->as_array_klass()->element_klass() != nullptr &&
                     k->as_array_klass()->element_klass()->is_inlinetype() &&
-                   !k->as_array_klass()->element_klass()->flat_in_array()));
+                   !k->as_array_klass()->element_klass()->maybe_flat_in_array()));
 
   return TypeAryKlassPtr::make(ptr, k, offset, interface_handling, not_flat, not_null_free, flat, null_free);
 }
@@ -6998,7 +6998,7 @@ const TypeKlassPtr *TypeAryKlassPtr::cast_to_exactness(bool klass_is_exact) cons
       const TypeOopPtr* exact_etype = TypeOopPtr::make_from_klass_unique(_elem->is_instklassptr()->instance_klass());
       bool not_inline = !exact_etype->can_be_inline_type();
       not_null_free = not_inline;
-      not_flat = !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->flat_in_array());
+      not_flat = !UseArrayFlattening || not_inline || (exact_etype->is_inlinetypeptr() && !exact_etype->inline_klass()->maybe_flat_in_array());
     }
   }
   return make(klass_is_exact ? Constant : NotNull, elem, k, _offset, not_flat, not_null_free, _flat, _null_free);

src/hotspot/share/prims/jvm.cpp

@@ -486,7 +486,7 @@ JVM_ENTRY(jarray, JVM_NewNullRestrictedNonAtomicArray(JNIEnv *env, jclass elmCla
   validate_array_arguments(klass, len, CHECK_NULL);
   InlineKlass* vk = InlineKlass::cast(klass);
   oop array = nullptr;
-  if (vk->flat_array() && vk->has_non_atomic_layout()) {
+  if (vk->maybe_flat_in_array() && vk->has_non_atomic_layout()) {
     array = oopFactory::new_flatArray(vk, len, LayoutKind::NON_ATOMIC_FLAT, CHECK_NULL);
     for (int i = 0; i < len; i++) {
       ((flatArrayOop)array)->write_value_to_flat_array(init_h(), i, CHECK_NULL);

src/hotspot/share/runtime/deoptimization.cpp

@@ -1661,7 +1661,7 @@ static int reassign_fields_by_klass(InstanceKlass* klass, frame* fr, RegisterMap
 // restore fields of an eliminated inline type array
 void Deoptimization::reassign_flat_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, flatArrayOop obj, FlatArrayKlass* vak, bool is_jvmci, TRAPS) {
   InlineKlass* vk = vak->element_klass();
-  assert(vk->flat_array(), "should only be used for flat inline type arrays");
+  assert(vk->maybe_flat_in_array(), "should only be used for flat inline type arrays");
   // Adjust offset to omit oop header
   int base_offset = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT) - InlineKlass::cast(vk)->payload_offset();
   // Initialize all elements of the flat inline type array

test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrays.java

@@ -3625,4 +3625,30 @@ static String test150(String s) {
     public void test150_verifier() {
         Asserts.assertEquals(test150("bla"), "bla");
     }
+
+    static value class Test151Value {
+        byte b;
+        String s;
+
+        Test151Value(byte b, String s) {
+            this.b = b;
+            this.s = s;
+        }
+
+        static final Test151Value DEFAULT = new Test151Value((byte) 1, "hello");
+
+        static final Test151Value[] ARRAY = (Test151Value[]) ValueClass.newNullRestrictedAtomicArray(Test151Value.class, 100, DEFAULT);
+    }
+
+    @Test
+    @IR(applyIf = {"InlineTypeReturnedAsFields", "true"},
+        failOn = {ALLOC})
+    static Test151Value test151(int i) {
+        return Test151Value.ARRAY[i];
+    }
+
+    @Run(test = "test151")
+    public void test151_verifier() {
+        Asserts.assertEquals(Test151Value.DEFAULT, test151(rI & 15));
+    }
 }