poly1305: add optimized s390x SIMD implementation with VMSL

SIMD implementation based the on the algorithm outlined in:
NEON crypto, Daniel J. Bernstein and Peter Schwabe
https://cryptojedi.org/papers/neoncrypto-20120320.pdf
and as modified for VMSL as described in
Accelerating Poly1305 Cryptographic Message Authentication on the z14
O'Farrell, Gadriwala, et al, CASCON 2017, p48-55
https://ibm.ent.box.com/s/jf9gedj0e9d2vjctfyh186shaztavnht

name		old		new		delta
64		485MB/s		1315 MB/s	+171.58%
1K		607MB/s		4352 MB/s	+616.97%
64Unaligned	485MB/s		1373 MB/s       +183.09%
1KUnaligned	606MB/s		4286 MB/s	+607.26%
2M		607MB/s		5529 MB/s	+810.87%

Change-Id: I31ccc25ced09180d99ea5c9233f0dcdc8666fc98
Reviewed-on: https://go-review.googlesource.com/110297
Run-TryBot: Michael Munday <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Michael Munday <[email protected]>

Author: wgo

poly1305/poly1305_test.go

@@ -75,7 +75,7 @@ var testData = []struct {
 	},
 }
 
-func testSum(t *testing.T, unaligned bool) {
+func testSum(t *testing.T, unaligned bool, sumImpl func(tag *[TagSize]byte, msg []byte, key *[32]byte)) {
 	var out [16]byte
 	var key [32]byte
 
@@ -85,7 +85,7 @@ func testSum(t *testing.T, unaligned bool) {
 			in = unalignBytes(in)
 		}
 		copy(key[:], v.k)
-		Sum(&out, in, &key)
+		sumImpl(&out, in, &key)
 		if !bytes.Equal(out[:], v.correct) {
 			t.Errorf("%d: expected %x, got %x", i, v.correct, out[:])
 		}
@@ -125,8 +125,10 @@ func TestBurnin(t *testing.T) {
 	}
 }
 
-func TestSum(t *testing.T)          { testSum(t, false) }
-func TestSumUnaligned(t *testing.T) { testSum(t, true) }
+func TestSum(t *testing.T)                 { testSum(t, false, Sum) }
+func TestSumUnaligned(t *testing.T)        { testSum(t, true, Sum) }
+func TestSumGeneric(t *testing.T)          { testSum(t, false, sumGeneric) }
+func TestSumGenericUnaligned(t *testing.T) { testSum(t, true, sumGeneric) }
 
 func benchmark(b *testing.B, size int, unaligned bool) {
 	var out [16]byte
@@ -146,6 +148,7 @@ func Benchmark64(b *testing.B)          { benchmark(b, 64, false) }
 func Benchmark1K(b *testing.B)          { benchmark(b, 1024, false) }
 func Benchmark64Unaligned(b *testing.B) { benchmark(b, 64, true) }
 func Benchmark1KUnaligned(b *testing.B) { benchmark(b, 1024, true) }
+func Benchmark2M(b *testing.B)          { benchmark(b, 2097152, true) }
 
 func unalignBytes(in []byte) []byte {
 	out := make([]byte, len(in)+1)

poly1305/sum_noasm.go

@@ -0,0 +1,14 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build s390x,!go1.11 !arm,!amd64,!s390x gccgo appengine nacl
+
+package poly1305
+
+// Sum generates an authenticator for msg using a one-time key and puts the
+// 16-byte result into out. Authenticating two different messages with the same
+// key allows an attacker to forge messages at will.
+func Sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
+	sumGeneric(out, msg, key)
+}

poly1305/sum_ref.go

@@ -2,16 +2,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !amd64,!arm gccgo appengine nacl
-
 package poly1305
 
 import "encoding/binary"
 
-// Sum generates an authenticator for msg using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
+// sumGeneric generates an authenticator for msg using a one-time key and
+// puts the 16-byte result into out. This is the generic implementation of
+// Sum and should be called if no assembly implementation is available.
+func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) {
 	var (
 		h0, h1, h2, h3, h4 uint32 // the hash accumulators
 		r0, r1, r2, r3, r4 uint64 // the r part of the key

poly1305/sum_s390x.go

@@ -0,0 +1,49 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build s390x,go1.11,!gccgo,!appengine
+
+package poly1305
+
+// hasVectorFacility reports whether the machine supports
+// the vector facility (vx).
+func hasVectorFacility() bool
+
+// hasVMSLFacility reports whether the machine supports
+// Vector Multiply Sum Logical (VMSL).
+func hasVMSLFacility() bool
+
+var hasVX = hasVectorFacility()
+var hasVMSL = hasVMSLFacility()
+
+// poly1305vx is an assembly implementation of Poly1305 that uses vector
+// instructions. It must only be called if the vector facility (vx) is
+// available.
+//go:noescape
+func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+
+// poly1305vmsl is an assembly implementation of Poly1305 that uses vector
+// instructions, including VMSL. It must only be called if the vector facility (vx) is
+// available and if VMSL is supported.
+//go:noescape
+func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
+
+// Sum generates an authenticator for m using a one-time key and puts the
+// 16-byte result into out. Authenticating two different messages with the same
+// key allows an attacker to forge messages at will.
+func Sum(out *[16]byte, m []byte, key *[32]byte) {
+	if hasVX {
+		var mPtr *byte
+		if len(m) > 0 {
+			mPtr = &m[0]
+		}
+		if hasVMSL && len(m) > 256 {
+			poly1305vmsl(out, mPtr, uint64(len(m)), key)
+		} else {
+			poly1305vx(out, mPtr, uint64(len(m)), key)
+		}
+	} else {
+		sumGeneric(out, m, key)
+	}
+}