Fix creating lists and tuples from strings with surrogates

Author: msimacek

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/StringUtils.java

@@ -40,7 +40,7 @@
  */
 package com.oracle.graal.python.builtins.objects.str;
 
-import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 
 public final class StringUtils {
     public enum StripKind {
@@ -203,8 +203,24 @@ public static String strip(String str, String chars, StripKind stripKind) {
         return str.substring(i, j);
     }
 
-    @CompilerDirectives.TruffleBoundary
+    @TruffleBoundary
     public static boolean containsNullCharacter(String value) {
         return value.indexOf(0) > 0;
     }
+
+    @TruffleBoundary
+    public static Object[] toCharacterArray(String arg) {
+        Object[] values = new Object[arg.codePointCount(0, arg.length())];
+        for (int i = 0, o = 0; i < arg.length(); o++) {
+            int codePoint = arg.codePointAt(i);
+            int charCount = Character.charCount(codePoint);
+            if (charCount == 1) {
+                values[o] = String.valueOf((char) codePoint);
+            } else {
+                values[o] = String.valueOf(Character.toChars(codePoint));
+            }
+            i += charCount;
+        }
+        return values;
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/builtins/ListNodes.java

@@ -53,7 +53,7 @@
 import com.oracle.graal.python.builtins.objects.list.PList;
 import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
 import com.oracle.graal.python.builtins.objects.slice.PSlice;
-import com.oracle.graal.python.builtins.objects.str.PString;
+import com.oracle.graal.python.builtins.objects.str.StringUtils;
 import com.oracle.graal.python.nodes.ErrorMessages;
 import com.oracle.graal.python.nodes.PGuards;
 import com.oracle.graal.python.nodes.PNodeWithContext;
@@ -95,25 +95,10 @@ public final PList execute(Object value) {
 
         protected abstract PList execute(Object cls, Object value);
 
-        @Specialization
-        PList listString(Object cls, PString arg,
-                        @Shared("appendNode") @Cached AppendNode appendNode,
-                        @Shared("factory") @Cached PythonObjectFactory factory) {
-            return listString(cls, arg.getValue(), appendNode, factory);
-        }
-
         @Specialization
         PList listString(Object cls, String arg,
-                        @Shared("appendNode") @Cached AppendNode appendNode,
                         @Shared("factory") @Cached PythonObjectFactory factory) {
-            char[] chars = arg.toCharArray();
-            PList list = factory.createList(cls);
-
-            for (char c : chars) {
-                appendNode.execute(list, Character.toString(c));
-            }
-
-            return list;
+            return factory.createList(cls, StringUtils.toCharacterArray(arg));
         }
 
         @Specialization(guards = "isNoValue(none)")

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/builtins/TupleNodes.java

@@ -44,6 +44,7 @@
 import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.CreateStorageFromIteratorNode;
 import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
+import com.oracle.graal.python.builtins.objects.str.StringUtils;
 import com.oracle.graal.python.builtins.objects.tuple.PTuple;
 import com.oracle.graal.python.nodes.PGuards;
 import com.oracle.graal.python.nodes.PNodeWithContext;
@@ -79,11 +80,7 @@ PTuple tuple(Object cls, @SuppressWarnings("unused") PNone none) {
 
         @Specialization
         PTuple tuple(Object cls, String arg) {
-            Object[] values = new Object[arg.length()];
-            for (int i = 0; i < arg.length(); i++) {
-                values[i] = String.valueOf(arg.charAt(i));
-            }
-            return factory.createTuple(cls, values);
+            return factory.createTuple(cls, StringUtils.toCharacterArray(arg));
         }
 
         @Specialization(guards = {"cannotBeOverridden(cls)", "cannotBeOverridden(plib.getLazyPythonClass(iterable))"}, limit = "2")

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/frame/DestructuringAssignmentNode.java

@@ -27,7 +27,6 @@
 
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
 
-import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
 import com.oracle.graal.python.builtins.objects.list.PList;
@@ -38,14 +37,10 @@
 import com.oracle.graal.python.nodes.expression.ExpressionNode;
 import com.oracle.graal.python.nodes.object.IsBuiltinClassProfile;
 import com.oracle.graal.python.nodes.statement.StatementNode;
-import com.oracle.graal.python.runtime.PythonContext;
-import com.oracle.graal.python.runtime.PythonCore;
 import com.oracle.graal.python.runtime.object.PythonObjectFactory;
 import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
 import com.oracle.truffle.api.CompilerAsserts;
 import com.oracle.truffle.api.CompilerDirectives;
-import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
-import com.oracle.truffle.api.TruffleLanguage.ContextReference;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Cached.Shared;
 import com.oracle.truffle.api.dsl.Specialization;
@@ -56,7 +51,6 @@
 public abstract class DestructuringAssignmentNode extends StatementNode implements WriteNode {
     /* Lazily initialized helpers, also acting as branch profiles */
     @Child private PRaiseNode raiseNode;
-    @CompilationFinal private ContextReference<PythonContext> contextRef;
 
     /* Syntactic children */
     @Child private ExpressionNode rhs;
@@ -128,7 +122,7 @@ private void writeSequenceStorage(VirtualFrame frame, SequenceStorage sequenceSt
         int len = lenNode.execute(sequenceStorage);
         if (len > slots.length) {
             CompilerDirectives.transferToInterpreter();
-            throw getCore().raise(ValueError, ErrorMessages.TOO_MANY_VALUES_TO_UNPACK, slots.length);
+            throw ensureRaiseNode().raise(ValueError, ErrorMessages.TOO_MANY_VALUES_TO_UNPACK, slots.length);
         } else if (len < slots.length) {
             throw ensureRaiseNode().raise(ValueError, ErrorMessages.NOT_ENOUGH_VALUES_TO_UNPACK, slots.length, len);
         } else {
@@ -187,14 +181,6 @@ void writeIterableStarred(VirtualFrame frame, Object iterable,
         performAssignments(frame);
     }
 
-    private PythonCore getCore() {
-        if (contextRef == null) {
-            CompilerDirectives.transferToInterpreterAndInvalidate();
-            contextRef = lookupContextReference(PythonLanguage.class);
-        }
-        return contextRef.get().getCore();
-    }
-
     private PRaiseNode ensureRaiseNode() {
         if (raiseNode == null) {
             CompilerDirectives.transferToInterpreterAndInvalidate();