Initial work to improve the state of encodings for templates

Author: wycats

actionpack/lib/action_view.rb

@@ -58,6 +58,8 @@ module ActionView
   end
 
   autoload :TestCase, 'action_view/test_case'
+
+  ENCODING_FLAG = "#.*coding[:=]\s*(\S+)[ \t]*"
 end
 
 require 'active_support/i18n'

actionpack/lib/action_view/template.rb

@@ -27,6 +27,12 @@ class Template
     end
 
     def initialize(source, identifier, handler, details)
+      if source.encoding_aware? && source =~ %r{\A#{ENCODING_FLAG}}
+        # don't snip off the \n to preserve line numbers
+        source.sub!(/\A[^\n]*/, '')
+        source.force_encoding($1).encode
+      end
+
       @source     = source
       @identifier = identifier
       @handler    = handler

actionpack/lib/action_view/template/handlers/erb.rb

@@ -1,5 +1,6 @@
 require 'active_support/core_ext/class/attribute_accessors'
 require 'active_support/core_ext/string/output_safety'
+require "action_view/template"
 require 'erubis'
 
 module ActionView
@@ -17,65 +18,72 @@ def append_if_string=(value)
     end
   end
 
-  module Template::Handlers
-    class Erubis < ::Erubis::Eruby
-      def add_preamble(src)
-        src << "@output_buffer = ActionView::OutputBuffer.new;"
-      end
+  class Template
+    module Handlers
+      class Erubis < ::Erubis::Eruby
+        def add_preamble(src)
+          src << "@output_buffer = ActionView::OutputBuffer.new;"
+        end
 
-      def add_text(src, text)
-        return if text.empty?
-        src << "@output_buffer.safe_concat('" << escape_text(text) << "');"
-      end
+        def add_text(src, text)
+          return if text.empty?
+          src << "@output_buffer.safe_concat('" << escape_text(text) << "');"
+        end
 
-      BLOCK_EXPR = /\s+(do|\{)(\s*\|[^|]*\|)?\s*\Z/
+        BLOCK_EXPR = /\s+(do|\{)(\s*\|[^|]*\|)?\s*\Z/
 
-      def add_expr_literal(src, code)
-        if code =~ BLOCK_EXPR
-          src << '@output_buffer.append= ' << code
-        else
-          src << '@output_buffer.append= (' << code << ');'
+        def add_expr_literal(src, code)
+          if code =~ BLOCK_EXPR
+            src << '@output_buffer.append= ' << code
+          else
+            src << '@output_buffer.append= (' << code << ');'
+          end
         end
-      end
 
-      def add_stmt(src, code)
-        if code =~ BLOCK_EXPR
-          src << '@output_buffer.append_if_string= ' << code
-        else
-          super
+        def add_stmt(src, code)
+          if code =~ BLOCK_EXPR
+            src << '@output_buffer.append_if_string= ' << code
+          else
+            super
+          end
         end
-      end
 
-      def add_expr_escaped(src, code)
-        src << '@output_buffer.append= ' << escaped_expr(code) << ';'
-      end
+        def add_expr_escaped(src, code)
+          src << '@output_buffer.append= ' << escaped_expr(code) << ';'
+        end
 
-      def add_postamble(src)
-        src << '@output_buffer.to_s'
+        def add_postamble(src)
+          src << '@output_buffer.to_s'
+        end
       end
-    end
 
-    class ERB < Template::Handler
-      include Compilable
+      class ERB < Handler
+        include Compilable
 
-      ##
-      # :singleton-method:
-      # Specify trim mode for the ERB compiler. Defaults to '-'.
-      # See ERb documentation for suitable values.
-      cattr_accessor :erb_trim_mode
-      self.erb_trim_mode = '-'
+        ##
+        # :singleton-method:
+        # Specify trim mode for the ERB compiler. Defaults to '-'.
+        # See ERb documentation for suitable values.
+        cattr_accessor :erb_trim_mode
+        self.erb_trim_mode = '-'
 
-      self.default_format = Mime::HTML
+        self.default_format = Mime::HTML
 
-      cattr_accessor :erb_implementation
-      self.erb_implementation = Erubis
+        cattr_accessor :erb_implementation
+        self.erb_implementation = Erubis
 
-      def compile(template)
-        source = template.source.gsub(/\A(<%(#.*coding[:=]\s*(\S+)\s*)-?%>)\s*\n?/, '')
-        erb = "<% __in_erb_template=true %>#{source}"
-        result = self.class.erb_implementation.new(erb, :trim=>(self.class.erb_trim_mode == "-")).src
-        result = "#{$2}\n#{result}" if $2
-        result
+        ENCODING_TAG = Regexp.new("\A(<%#{ENCODING_FLAG}-?%>)[ \t]*")
+
+        def compile(template)
+          erb = template.source.gsub(ENCODING_TAG, '')
+          result = self.class.erb_implementation.new(
+            erb,
+            :trim => (self.class.erb_trim_mode == "-")
+          ).src
+
+          result = "#{$2}\n#{result}" if $2
+          result
+        end
       end
     end
   end

activesupport/lib/active_support/core_ext/string/encoding.rb

@@ -0,0 +1,11 @@
+class String
+  if defined?(Encoding) && "".respond_to?(:encode)
+    def encoding_aware?
+      true
+    end
+  else
+    def encoding_aware?
+      false
+    end
+  end
+end

activesupport/lib/active_support/ruby/shim.rb

@@ -15,6 +15,7 @@
 require 'active_support/core_ext/process/daemon'
 require 'active_support/core_ext/string/conversions'
 require 'active_support/core_ext/string/interpolation'
+require 'active_support/core_ext/string/encoding'
 require 'active_support/core_ext/rexml'
 require 'active_support/core_ext/time/conversions'
 require 'active_support/core_ext/file/path'

activesupport/test/core_ext/string_ext_test.rb

@@ -439,6 +439,14 @@ def to_s
   test 'emits normal string yaml' do
     assert_equal 'foo'.to_yaml, 'foo'.html_safe.to_yaml(:foo => 1)
   end
+
+  test 'knows whether it is encoding aware' do
+    if RUBY_VERSION >= "1.9"
+      assert 'ruby'.encoding_aware?
+    else
+      assert !'ruby'.encoding_aware?
+    end
+  end
 end
 
 class StringExcludeTest < ActiveSupport::TestCase